Add hardening support

Add charmhelpers.contrib.hardening and calls to install,
config-changed, upgrade-charm and update-status hooks.
Also add new config option to allow one or more hardening
modules to be applied at runtime.

Change-Id: I0f3035c8f8feae90ad3572297fab0ac28e7d97e2

charm-helpers-hooks.yaml

@@ -12,3 +12,4 @@ include:
     - contrib.python
     - payload
     - contrib.charmsupport
+    - contrib.hardening|inc=*

config.yaml

@@ -226,3 +226,9 @@ options:
       wait for you to execute the openstack-upgrade action for this charm on
       each unit. If False it will revert to existing behavior of upgrading
       all units on config change.
+  harden:
+    default:
+    type: string
+    description: |
+      Apply system hardening. Supports a space-delimited list of modules
+      to run. Supported modules currently include os, ssh, apache and mysql.

hardening.yaml

@@ -0,0 +1,5 @@
+# Overrides file for contrib.hardening. See README.hardening in
+# contrib.hardening for info on how to use this file.
+ssh:
+  server:
+    use_pam: 'yes' # juju requires this

hooks/charmhelpers/contrib/hardening/README.hardening.md

@@ -0,0 +1,38 @@
+# Juju charm-helpers hardening library
+
+## Description
+
+This library provides multiple implementations of system and application
+hardening that conform to the standards of http://hardening.io/.
+
+Current implementations include:
+
+ * OS
+ * SSH
+ * MySQL
+ * Apache
+
+## Requirements
+
+* Juju Charms
+
+## Usage
+
+1. Synchronise this library into your charm and add the harden() decorator
+   (from contrib.hardening.harden) to any functions or methods you want to use
+   to trigger hardening of your application/system.
+
+2. Add a config option called 'harden' to your charm config.yaml and set it to
+   a space-delimited list of hardening modules you want to run e.g. "os ssh"
+
+3. Override any config defaults (contrib.hardening.defaults) by adding a file
+   called hardening.yaml to your charm root containing the name(s) of the
+   modules whose settings you want override at root level and then any settings
+   with overrides e.g.
+   
+   os:
+       general:
+            desktop_enable: True
+
+4. Now just run your charm as usual and hardening will be applied each time the
+   hook runs.

hooks/charmhelpers/contrib/hardening/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

hooks/charmhelpers/contrib/hardening/apache/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from os import path
+
+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')

hooks/charmhelpers/contrib/hardening/apache/checks/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+)
+from charmhelpers.contrib.hardening.apache.checks import config
+
+
+def run_apache_checks():
+    log("Starting Apache hardening checks.", level=DEBUG)
+    checks = config.get_audits()
+    for check in checks:
+        log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
+        check.ensure_compliance()
+
+    log("Apache hardening checks complete.", level=DEBUG)

hooks/charmhelpers/contrib/hardening/apache/checks/config.py

@@ -0,0 +1,100 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import subprocess
+
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO,
+)
+from charmhelpers.contrib.hardening.audits.file import (
+    FilePermissionAudit,
+    DirectoryPermissionAudit,
+    NoReadWriteForOther,
+    TemplatedFile,
+)
+from charmhelpers.contrib.hardening.audits.apache import DisabledModuleAudit
+from charmhelpers.contrib.hardening.apache import TEMPLATES_DIR
+from charmhelpers.contrib.hardening import utils
+
+
+def get_audits():
+    """Get Apache hardening config audits.
+
+    :returns:  dictionary of audits
+    """
+    if subprocess.call(['which', 'apache2'], stdout=subprocess.PIPE) != 0:
+        log("Apache server does not appear to be installed on this node - "
+            "skipping apache hardening", level=INFO)
+        return []
+
+    context = ApacheConfContext()
+    settings = utils.get_settings('apache')
+    audits = [
+        FilePermissionAudit(paths='/etc/apache2/apache2.conf', user='root',
+                            group='root', mode=0o0640),
+
+        TemplatedFile(os.path.join(settings['common']['apache_dir'],
+                                   'mods-available/alias.conf'),
+                      context,
+                      TEMPLATES_DIR,
+                      mode=0o0755,
+                      user='root',
+                      service_actions=[{'service': 'apache2',
+                                        'actions': ['restart']}]),
+
+        TemplatedFile(os.path.join(settings['common']['apache_dir'],
+                                   'conf-enabled/hardening.conf'),
+                      context,
+                      TEMPLATES_DIR,
+                      mode=0o0640,
+                      user='root',
+                      service_actions=[{'service': 'apache2',
+                                        'actions': ['restart']}]),
+
+        DirectoryPermissionAudit(settings['common']['apache_dir'],
+                                 user='root',
+                                 group='root',
+                                 mode=0o640),
+
+        DisabledModuleAudit(settings['hardening']['modules_to_disable']),
+
+        NoReadWriteForOther(settings['common']['apache_dir']),
+    ]
+
+    return audits
+
+
+class ApacheConfContext(object):
+    """Defines the set of key/value pairs to set in a apache config file.
+
+    This context, when called, will return a dictionary containing the
+    key/value pairs of setting to specify in the
+    /etc/apache/conf-enabled/hardening.conf file.
+    """
+    def __call__(self):
+        settings = utils.get_settings('apache')
+        ctxt = settings['hardening']
+
+        out = subprocess.check_output(['apache2', '-v'])
+        ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
+                                           out).group(1)
+        ctxt['apache_icondir'] = '/usr/share/apache2/icons/'
+        ctxt['traceenable'] = settings['hardening']['traceenable']
+        return ctxt

hooks/charmhelpers/contrib/hardening/apache/templates/alias.conf

@@ -0,0 +1,31 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+<IfModule alias_module>
+  #
+  # Aliases: Add here as many aliases as you need (with no limit). The format is
+  # Alias fakename realname
+  #
+  # Note that if you include a trailing / on fakename then the server will
+  # require it to be present in the URL.  So "/icons" isn't aliased in this
+  # example, only "/icons/".  If the fakename is slash-terminated, then the
+  # realname must also be slash terminated, and if the fakename omits the
+  # trailing slash, the realname must also omit it.
+  #
+  # We include the /icons/ alias for FancyIndexed directory listings.  If
+  # you do not use FancyIndexing, you may comment this out.
+  #
+  Alias /icons/ "{{ apache_icondir }}/"
+
+  <Directory "{{ apache_icondir }}">
+    Options -Indexes -MultiViews -FollowSymLinks
+    AllowOverride None
+{% if apache_version == '2.4' -%}
+    Require all granted
+{% else -%}
+    Order allow,deny
+    Allow from all
+{% endif %}
+  </Directory>
+</IfModule>

hooks/charmhelpers/contrib/hardening/apache/templates/hardening.conf

@@ -0,0 +1,18 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+
+<Location / >
+  <LimitExcept {{ allowed_http_methods }} > 
+    # http://httpd.apache.org/docs/2.4/upgrading.html
+    {% if apache_version > '2.2' -%}
+    Require all granted
+    {% else -%}
+    Order Allow,Deny 
+    Deny from all 
+    {% endif %}
+  </LimitExcept>
+</Location>
+
+TraceEnable {{ traceenable }}

hooks/charmhelpers/contrib/hardening/audits/__init__.py

@@ -0,0 +1,63 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class BaseAudit(object):  # NO-QA
+    """Base class for hardening checks.
+
+    The lifecycle of a hardening check is to first check to see if the system
+    is in compliance for the specified check. If it is not in compliance, the
+    check method will return a value which will be supplied to the.
+    """
+    def __init__(self, *args, **kwargs):
+        self.unless = kwargs.get('unless', None)
+        super(BaseAudit, self).__init__()
+
+    def ensure_compliance(self):
+        """Checks to see if the current hardening check is in compliance or
+        not.
+
+        If the check that is performed is not in compliance, then an exception
+        should be raised.
+        """
+        pass
+
+    def _take_action(self):
+        """Determines whether to perform the action or not.
+
+        Checks whether or not an action should be taken. This is determined by
+        the truthy value for the unless parameter. If unless is a callback
+        method, it will be invoked with no parameters in order to determine
+        whether or not the action should be taken. Otherwise, the truthy value
+        of the unless attribute will determine if the action should be
+        performed.
+        """
+        # Do the action if there isn't an unless override.
+        if self.unless is None:
+            return True
+
+        # Invoke the callback if there is one.
+        if hasattr(self.unless, '__call__'):
+            results = self.unless()
+            if results:
+                return False
+            else:
+                return True
+
+        if self.unless:
+            return False
+        else:
+            return True

hooks/charmhelpers/contrib/hardening/audits/apache.py

@@ -0,0 +1,100 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import subprocess
+
+from six import string_types
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO,
+    ERROR,
+)
+
+from charmhelpers.contrib.hardening.audits import BaseAudit
+
+
+class DisabledModuleAudit(BaseAudit):
+    """Audits Apache2 modules.
+
+    Determines if the apache2 modules are enabled. If the modules are enabled
+    then they are removed in the ensure_compliance.
+    """
+    def __init__(self, modules):
+        if modules is None:
+            self.modules = []
+        elif isinstance(modules, string_types):
+            self.modules = [modules]
+        else:
+            self.modules = modules
+
+    def ensure_compliance(self):
+        """Ensures that the modules are not loaded."""
+        if not self.modules:
+            return
+
+        try:
+            loaded_modules = self._get_loaded_modules()
+            non_compliant_modules = []
+            for module in self.modules:
+                if module in loaded_modules:
+                    log("Module '%s' is enabled but should not be." %
+                        (module), level=INFO)
+                    non_compliant_modules.append(module)
+
+            if len(non_compliant_modules) == 0:
+                return
+
+            for module in non_compliant_modules:
+                self._disable_module(module)
+            self._restart_apache()
+        except subprocess.CalledProcessError as e:
+            log('Error occurred auditing apache module compliance. '
+                'This may have been already reported. '
+                'Output is: %s' % e.output, level=ERROR)
+
+    @staticmethod
+    def _get_loaded_modules():
+        """Returns the modules which are enabled in Apache."""
+        output = subprocess.check_output(['apache2ctl', '-M'])
+        modules = []
+        for line in output.strip().split():
+            # Each line of the enabled module output looks like:
+            #  module_name (static|shared)
+            # Plus a header line at the top of the output which is stripped
+            # out by the regex.
+            matcher = re.search(r'^ (\S*)', line)
+            if matcher:
+                modules.append(matcher.group(1))
+        return modules
+
+    @staticmethod
+    def _disable_module(module):
+        """Disables the specified module in Apache."""
+        try:
+            subprocess.check_call(['a2dismod', module])
+        except subprocess.CalledProcessError as e:
+            # Note: catch error here to allow the attempt of disabling
+            # multiple modules in one go rather than failing after the
+            # first module fails.
+            log('Error occurred disabling module %s. '
+                'Output is: %s' % (module, e.output), level=ERROR)
+
+    @staticmethod
+    def _restart_apache():
+        """Restarts the apache process"""
+        subprocess.check_output(['service', 'apache2', 'restart'])

hooks/charmhelpers/contrib/hardening/audits/apt.py

@@ -0,0 +1,105 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import  # required for external apt import
+from apt import apt_pkg
+from six import string_types
+
+from charmhelpers.fetch import (
+    apt_cache,
+    apt_purge
+)
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+    WARNING,
+)
+from charmhelpers.contrib.hardening.audits import BaseAudit
+
+
+class AptConfig(BaseAudit):
+
+    def __init__(self, config, **kwargs):
+        self.config = config
+
+    def verify_config(self):
+        apt_pkg.init()
+        for cfg in self.config:
+            value = apt_pkg.config.get(cfg['key'], cfg.get('default', ''))
+            if value and value != cfg['expected']:
+                log("APT config '%s' has unexpected value '%s' "
+                    "(expected='%s')" %
+                    (cfg['key'], value, cfg['expected']), level=WARNING)
+
+    def ensure_compliance(self):
+        self.verify_config()
+
+
+class RestrictedPackages(BaseAudit):
+    """Class used to audit restricted packages on the system."""
+
+    def __init__(self, pkgs, **kwargs):
+        super(RestrictedPackages, self).__init__(**kwargs)
+        if isinstance(pkgs, string_types) or not hasattr(pkgs, '__iter__'):
+            self.pkgs = [pkgs]
+        else:
+            self.pkgs = pkgs
+
+    def ensure_compliance(self):
+        cache = apt_cache()
+
+        for p in self.pkgs:
+            if p not in cache:
+                continue
+
+            pkg = cache[p]
+            if not self.is_virtual_package(pkg):
+                if not pkg.current_ver:
+                    log("Package '%s' is not installed." % pkg.name,
+                        level=DEBUG)
+                    continue
+                else:
+                    log("Restricted package '%s' is installed" % pkg.name,
+                        level=WARNING)
+                    self.delete_package(cache, pkg)
+            else:
+                log("Checking restricted virtual package '%s' provides" %
+                    pkg.name, level=DEBUG)
+                self.delete_package(cache, pkg)
+
+    def delete_package(self, cache, pkg):
+        """Deletes the package from the system.
+
+        Deletes the package form the system, properly handling virtual
+        packages.
+
+        :param cache: the apt cache
+        :param pkg: the package to remove
+        """
+        if self.is_virtual_package(pkg):
+            log("Package '%s' appears to be virtual - purging provides" %
+                pkg.name, level=DEBUG)
+            for _p in pkg.provides_list:
+                self.delete_package(cache, _p[2].parent_pkg)
+        elif not pkg.current_ver:
+            log("Package '%s' not installed" % pkg.name, level=DEBUG)
+            return
+        else:
+            log("Purging package '%s'" % pkg.name, level=DEBUG)
+            apt_purge(pkg.name)
+
+    def is_virtual_package(self, pkg):
+        return pkg.has_provides and not pkg.has_versions

hooks/charmhelpers/contrib/hardening/audits/file.py

@@ -0,0 +1,552 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import grp
+import os
+import pwd
+import re
+
+from subprocess import (
+    CalledProcessError,
+    check_output,
+    check_call,
+)
+from traceback import format_exc
+from six import string_types
+from stat import (
+    S_ISGID,
+    S_ISUID
+)
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+    INFO,
+    WARNING,
+    ERROR,
+)
+from charmhelpers.core import unitdata
+from charmhelpers.core.host import file_hash
+from charmhelpers.contrib.hardening.audits import BaseAudit
+from charmhelpers.contrib.hardening.templating import (
+    get_template_path,
+    render_and_write,
+)
+from charmhelpers.contrib.hardening import utils
+
+
+class BaseFileAudit(BaseAudit):
+    """Base class for file audits.
+
+    Provides api stubs for compliance check flow that must be used by any class
+    that implemented this one.
+    """
+
+    def __init__(self, paths, always_comply=False, *args, **kwargs):
+        """
+        :param paths: string path of list of paths of files we want to apply
+                      compliance checks are criteria to.
+        :param always_comply: if true compliance criteria is always applied
+                              else compliance is skipped for non-existent
+                              paths.
+        """
+        super(BaseFileAudit, self).__init__(*args, **kwargs)
+        self.always_comply = always_comply
+        if isinstance(paths, string_types) or not hasattr(paths, '__iter__'):
+            self.paths = [paths]
+        else:
+            self.paths = paths
+
+    def ensure_compliance(self):
+        """Ensure that the all registered files comply to registered criteria.
+        """
+        for p in self.paths:
+            if os.path.exists(p):
+                if self.is_compliant(p):
+                    continue
+
+                log('File %s is not in compliance.' % p, level=INFO)
+            else:
+                if not self.always_comply:
+                    log("Non-existent path '%s' - skipping compliance check"
+                        % (p), level=INFO)
+                    continue
+
+            if self._take_action():
+                log("Applying compliance criteria to '%s'" % (p), level=INFO)
+                self.comply(p)
+
+    def is_compliant(self, path):
+        """Audits the path to see if it is compliance.
+
+        :param path: the path to the file that should be checked.
+        """
+        raise NotImplementedError
+
+    def comply(self, path):
+        """Enforces the compliance of a path.
+
+        :param path: the path to the file that should be enforced.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def _get_stat(cls, path):
+        """Returns the Posix st_stat information for the specified file path.
+
+        :param path: the path to get the st_stat information for.
+        :returns: an st_stat object for the path or None if the path doesn't
+                  exist.
+        """
+        return os.stat(path)
+
+
+class FilePermissionAudit(BaseFileAudit):
+    """Implements an audit for file permissions and ownership for a user.
+
+    This class implements functionality that ensures that a specific user/group
+    will own the file(s) specified and that the permissions specified are
+    applied properly to the file.
+    """
+    def __init__(self, paths, user, group=None, mode=0o600, **kwargs):
+        self.user = user
+        self.group = group
+        self.mode = mode
+        super(FilePermissionAudit, self).__init__(paths, user, group, mode,
+                                                  **kwargs)
+
+    @property
+    def user(self):
+        return self._user
+
+    @user.setter
+    def user(self, name):
+        try:
+            user = pwd.getpwnam(name)
+        except KeyError:
+            log('Unknown user %s' % name, level=ERROR)
+            user = None
+        self._user = user
+
+    @property
+    def group(self):
+        return self._group
+
+    @group.setter
+    def group(self, name):
+        try:
+            group = None
+            if name:
+                group = grp.getgrnam(name)
+            else:
+                group = grp.getgrgid(self.user.pw_gid)
+        except KeyError:
+            log('Unknown group %s' % name, level=ERROR)
+        self._group = group
+
+    def is_compliant(self, path):
+        """Checks if the path is in compliance.
+
+        Used to determine if the path specified meets the necessary
+        requirements to be in compliance with the check itself.
+
+        :param path: the file path to check
+        :returns: True if the path is compliant, False otherwise.
+        """
+        stat = self._get_stat(path)
+        user = self.user
+        group = self.group
+
+        compliant = True
+        if stat.st_uid != user.pw_uid or stat.st_gid != group.gr_gid:
+            log('File %s is not owned by %s:%s.' % (path, user.pw_name,
+                                                    group.gr_name),
+                level=INFO)
+            compliant = False
+
+        # POSIX refers to the st_mode bits as corresponding to both the
+        # file type and file permission bits, where the least significant 12
+        # bits (o7777) are the suid (11), sgid (10), sticky bits (9), and the
+        # file permission bits (8-0)
+        perms = stat.st_mode & 0o7777
+        if perms != self.mode:
+            log('File %s has incorrect permissions, currently set to %s' %
+                (path, oct(stat.st_mode & 0o7777)), level=INFO)
+            compliant = False
+
+        return compliant
+
+    def comply(self, path):
+        """Issues a chown and chmod to the file paths specified."""
+        utils.ensure_permissions(path, self.user.pw_name, self.group.gr_name,
+                                 self.mode)
+
+
+class DirectoryPermissionAudit(FilePermissionAudit):
+    """Performs a permission check for the  specified directory path."""
+
+    def __init__(self, paths, user, group=None, mode=0o600,
+                 recursive=True, **kwargs):
+        super(DirectoryPermissionAudit, self).__init__(paths, user, group,
+                                                       mode, **kwargs)
+        self.recursive = recursive
+
+    def is_compliant(self, path):
+        """Checks if the directory is compliant.
+
+        Used to determine if the path specified and all of its children
+        directories are in compliance with the check itself.
+
+        :param path: the directory path to check
+        :returns: True if the directory tree is compliant, otherwise False.
+        """
+        if not os.path.isdir(path):
+            log('Path specified %s is not a directory.' % path, level=ERROR)
+            raise ValueError("%s is not a directory." % path)
+
+        if not self.recursive:
+            return super(DirectoryPermissionAudit, self).is_compliant(path)
+
+        compliant = True
+        for root, dirs, _ in os.walk(path):
+            if len(dirs) > 0:
+                continue
+
+            if not super(DirectoryPermissionAudit, self).is_compliant(root):
+                compliant = False
+                continue
+
+        return compliant
+
+    def comply(self, path):
+        for root, dirs, _ in os.walk(path):
+            if len(dirs) > 0:
+                super(DirectoryPermissionAudit, self).comply(root)
+
+
+class ReadOnly(BaseFileAudit):
+    """Audits that files and folders are read only."""
+    def __init__(self, paths, *args, **kwargs):
+        super(ReadOnly, self).__init__(paths=paths, *args, **kwargs)
+
+    def is_compliant(self, path):
+        try:
+            output = check_output(['find', path, '-perm', '-go+w',
+                                   '-type', 'f']).strip()
+
+            # The find above will find any files which have permission sets
+            # which allow too broad of write access. As such, the path is
+            # compliant if there is no output.
+            if output:
+                return False
+
+            return True
+        except CalledProcessError as e:
+            log('Error occurred checking finding writable files for %s. '
+                'Error information is: command %s failed with returncode '
+                '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output,
+                                           format_exc(e)), level=ERROR)
+            return False
+
+    def comply(self, path):
+        try:
+            check_output(['chmod', 'go-w', '-R', path])
+        except CalledProcessError as e:
+            log('Error occurred removing writeable permissions for %s. '
+                'Error information is: command %s failed with returncode '
+                '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output,
+                                           format_exc(e)), level=ERROR)
+
+
+class NoReadWriteForOther(BaseFileAudit):
+    """Ensures that the files found under the base path are readable or
+    writable by anyone other than the owner or the group.
+    """
+    def __init__(self, paths):
+        super(NoReadWriteForOther, self).__init__(paths)
+
+    def is_compliant(self, path):
+        try:
+            cmd = ['find', path, '-perm', '-o+r', '-type', 'f', '-o',
+                   '-perm', '-o+w', '-type', 'f']
+            output = check_output(cmd).strip()
+
+            # The find above here will find any files which have read or
+            # write permissions for other, meaning there is too broad of access
+            # to read/write the file. As such, the path is compliant if there's
+            # no output.
+            if output:
+                return False
+
+            return True
+        except CalledProcessError as e:
+            log('Error occurred while finding files which are readable or '
+                'writable to the world in %s. '
+                'Command output is: %s.' % (path, e.output), level=ERROR)
+
+    def comply(self, path):
+        try:
+            check_output(['chmod', '-R', 'o-rw', path])
+        except CalledProcessError as e:
+            log('Error occurred attempting to change modes of files under '
+                'path %s. Output of command is: %s' % (path, e.output))
+
+
+class NoSUIDSGIDAudit(BaseFileAudit):
+    """Audits that specified files do not have SUID/SGID bits set."""
+    def __init__(self, paths, *args, **kwargs):
+        super(NoSUIDSGIDAudit, self).__init__(paths=paths, *args, **kwargs)
+
+    def is_compliant(self, path):
+        stat = self._get_stat(path)
+        if (stat.st_mode & (S_ISGID | S_ISUID)) != 0:
+            return False
+
+        return True
+
+    def comply(self, path):
+        try:
+            log('Removing suid/sgid from %s.' % path, level=DEBUG)
+            check_output(['chmod', '-s', path])
+        except CalledProcessError as e:
+            log('Error occurred removing suid/sgid from %s.'
+                'Error information is: command %s failed with returncode '
+                '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output,
+                                           format_exc(e)), level=ERROR)
+
+
+class TemplatedFile(BaseFileAudit):
+    """The TemplatedFileAudit audits the contents of a templated file.
+
+    This audit renders a file from a template, sets the appropriate file
+    permissions, then generates a hashsum with which to check the content
+    changed.
+    """
+    def __init__(self, path, context, template_dir, mode, user='root',
+                 group='root', service_actions=None, **kwargs):
+        self.context = context
+        self.user = user
+        self.group = group
+        self.mode = mode
+        self.template_dir = template_dir
+        self.service_actions = service_actions
+        super(TemplatedFile, self).__init__(paths=path, always_comply=True,
+                                            **kwargs)
+
+    def is_compliant(self, path):
+        """Determines if the templated file is compliant.
+
+        A templated file is only compliant if it has not changed (as
+        determined by its sha256 hashsum) AND its file permissions are set
+        appropriately.
+
+        :param path: the path to check compliance.
+        """
+        same_templates = self.templates_match(path)
+        same_content = self.contents_match(path)
+        same_permissions = self.permissions_match(path)
+
+        if same_content and same_permissions and same_templates:
+            return True
+
+        return False
+
+    def run_service_actions(self):
+        """Run any actions on services requested."""
+        if not self.service_actions:
+            return
+
+        for svc_action in self.service_actions:
+            name = svc_action['service']
+            actions = svc_action['actions']
+            log("Running service '%s' actions '%s'" % (name, actions),
+                level=DEBUG)
+            for action in actions:
+                cmd = ['service', name, action]
+                try:
+                    check_call(cmd)
+                except CalledProcessError as exc:
+                    log("Service name='%s' action='%s' failed - %s" %
+                        (name, action, exc), level=WARNING)
+
+    def comply(self, path):
+        """Ensures the contents and the permissions of the file.
+
+        :param path: the path to correct
+        """
+        dirname = os.path.dirname(path)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+        self.pre_write()
+        render_and_write(self.template_dir, path, self.context())
+        utils.ensure_permissions(path, self.user, self.group, self.mode)
+        self.run_service_actions()
+        self.save_checksum(path)
+        self.post_write()
+
+    def pre_write(self):
+        """Invoked prior to writing the template."""
+        pass
+
+    def post_write(self):
+        """Invoked after writing the template."""
+        pass
+
+    def templates_match(self, path):
+        """Determines if the template files are the same.
+
+        The template file equality is determined by the hashsum of the
+        template files themselves. If there is no hashsum, then the content
+        cannot be sure to be the same so treat it as if they changed.
+        Otherwise, return whether or not the hashsums are the same.
+
+        :param path: the path to check
+        :returns: boolean
+        """
+        template_path = get_template_path(self.template_dir, path)
+        key = 'hardening:template:%s' % template_path
+        template_checksum = file_hash(template_path)
+        kv = unitdata.kv()
+        stored_tmplt_checksum = kv.get(key)
+        if not stored_tmplt_checksum:
+            kv.set(key, template_checksum)
+            kv.flush()
+            log('Saved template checksum for %s.' % template_path,
+                level=DEBUG)
+            # Since we don't have a template checksum, then assume it doesn't
+            # match and return that the template is different.
+            return False
+        elif stored_tmplt_checksum != template_checksum:
+            kv.set(key, template_checksum)
+            kv.flush()
+            log('Updated template checksum for %s.' % template_path,
+                level=DEBUG)
+            return False
+
+        # Here the template hasn't changed based upon the calculated
+        # checksum of the template and what was previously stored.
+        return True
+
+    def contents_match(self, path):
+        """Determines if the file content is the same.
+
+        This is determined by comparing hashsum of the file contents and
+        the saved hashsum. If there is no hashsum, then the content cannot
+        be sure to be the same so treat them as if they are not the same.
+        Otherwise, return True if the hashsums are the same, False if they
+        are not the same.
+
+        :param path: the file to check.
+        """
+        checksum = file_hash(path)
+
+        kv = unitdata.kv()
+        stored_checksum = kv.get('hardening:%s' % path)
+        if not stored_checksum:
+            # If the checksum hasn't been generated, return False to ensure
+            # the file is written and the checksum stored.
+            log('Checksum for %s has not been calculated.' % path, level=DEBUG)
+            return False
+        elif stored_checksum != checksum:
+            log('Checksum mismatch for %s.' % path, level=DEBUG)
+            return False
+
+        return True
+
+    def permissions_match(self, path):
+        """Determines if the file owner and permissions match.
+
+        :param path: the path to check.
+        """
+        audit = FilePermissionAudit(path, self.user, self.group, self.mode)
+        return audit.is_compliant(path)
+
+    def save_checksum(self, path):
+        """Calculates and saves the checksum for the path specified.
+
+        :param path: the path of the file to save the checksum.
+        """
+        checksum = file_hash(path)
+        kv = unitdata.kv()
+        kv.set('hardening:%s' % path, checksum)
+        kv.flush()
+
+
+class DeletedFile(BaseFileAudit):
+    """Audit to ensure that a file is deleted."""
+    def __init__(self, paths):
+        super(DeletedFile, self).__init__(paths)
+
+    def is_compliant(self, path):
+        return not os.path.exists(path)
+
+    def comply(self, path):
+        os.remove(path)
+
+
+class FileContentAudit(BaseFileAudit):
+    """Audit the contents of a file."""
+    def __init__(self, paths, cases, **kwargs):
+        # Cases we expect to pass
+        self.pass_cases = cases.get('pass', [])
+        # Cases we expect to fail
+        self.fail_cases = cases.get('fail', [])
+        super(FileContentAudit, self).__init__(paths, **kwargs)
+
+    def is_compliant(self, path):
+        """
+        Given a set of content matching cases i.e. tuple(regex, bool) where
+        bool value denotes whether or not regex is expected to match, check that
+        all cases match as expected with the contents of the file. Cases can be
+        expected to pass of fail.
+
+        :param path: Path of file to check.
+        :returns: Boolean value representing whether or not all cases are
+                  found to be compliant.
+        """
+        log("Auditing contents of file '%s'" % (path), level=DEBUG)
+        with open(path, 'r') as fd:
+            contents = fd.read()
+
+        matches = 0
+        for pattern in self.pass_cases:
+            key = re.compile(pattern, flags=re.MULTILINE)
+            results = re.search(key, contents)
+            if results:
+                matches += 1
+            else:
+                log("Pattern '%s' was expected to pass but instead it failed"
+                    % (pattern), level=WARNING)
+
+        for pattern in self.fail_cases:
+            key = re.compile(pattern, flags=re.MULTILINE)
+            results = re.search(key, contents)
+            if not results:
+                matches += 1
+            else:
+                log("Pattern '%s' was expected to fail but instead it passed"
+                    % (pattern), level=WARNING)
+
+        total = len(self.pass_cases) + len(self.fail_cases)
+        log("Checked %s cases and %s passed" % (total, matches), level=DEBUG)
+        return matches == total
+
+    def comply(self, *args, **kwargs):
+        """NOOP since we just issue warnings. This is to avoid the
+        NotImplememtedError.
+        """
+        log("Not applying any compliance criteria, only checks.", level=INFO)

hooks/charmhelpers/contrib/hardening/defaults/apache.yaml

@@ -0,0 +1,13 @@
+# NOTE: this file contains the default configuration for the 'apache' hardening
+#       code. If you want to override any settings you must add them to a file
+#       called hardening.yaml in the root directory of your charm using the
+#       name 'apache' as the root key followed by any of the following with new
+#       values.
+
+common:
+    apache_dir: '/etc/apache2'
+
+hardening:
+    traceenable: 'off'
+    allowed_http_methods: "GET POST"
+    modules_to_disable: [ cgi, cgid ]

hooks/charmhelpers/contrib/hardening/defaults/apache.yaml.schema

@@ -0,0 +1,9 @@
+# NOTE: this schema must contain all valid keys from it's associated defaults
+#       file. It is used to validate user-provided overrides.
+common:
+    apache_dir:
+    traceenable:
+
+hardening:
+    allowed_http_methods:
+    modules_to_disable:

hooks/charmhelpers/contrib/hardening/defaults/mysql.yaml

@@ -0,0 +1,38 @@
+# NOTE: this file contains the default configuration for the 'mysql' hardening
+#       code. If you want to override any settings you must add them to a file
+#       called hardening.yaml in the root directory of your charm using the
+#       name 'mysql' as the root key followed by any of the following with new
+#       values.
+
+hardening:
+    mysql-conf: /etc/mysql/my.cnf
+    hardening-conf: /etc/mysql/conf.d/hardening.cnf
+
+security:
+    # @see http://www.symantec.com/connect/articles/securing-mysql-step-step
+    # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_chroot
+    chroot: None
+
+    # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_safe-user-create
+    safe-user-create: 1
+
+    # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_secure-auth
+    secure-auth: 1
+
+    # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_symbolic-links
+    skip-symbolic-links: 1
+
+    # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_skip-show-database
+    skip-show-database: True
+
+    # @see http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_local_infile
+    local-infile: 0
+
+    # @see https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_allow-suspicious-udfs
+    allow-suspicious-udfs: 0
+
+    # @see https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_automatic_sp_privileges
+    automatic-sp-privileges: 0
+
+    # @see https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_secure-file-priv
+    secure-file-priv: /tmp

hooks/charmhelpers/contrib/hardening/defaults/mysql.yaml.schema

@@ -0,0 +1,15 @@
+# NOTE: this schema must contain all valid keys from it's associated defaults
+#       file. It is used to validate user-provided overrides.
+hardening:
+    mysql-conf:
+    hardening-conf:
+security:
+    chroot:
+    safe-user-create:
+    secure-auth:
+    skip-symbolic-links:
+    skip-show-database:
+    local-infile:
+    allow-suspicious-udfs:
+    automatic-sp-privileges:
+    secure-file-priv:

hooks/charmhelpers/contrib/hardening/defaults/os.yaml

@@ -0,0 +1,67 @@
+# NOTE: this file contains the default configuration for the 'os' hardening
+#       code. If you want to override any settings you must add them to a file
+#       called hardening.yaml in the root directory of your charm using the
+#       name 'os' as the root key followed by any of the following with new
+#       values.
+
+general:
+    desktop_enable: False  # (type:boolean)
+
+environment:
+    extra_user_paths: []
+    umask: 027
+    root_path: /
+
+auth:
+    pw_max_age: 60
+    # discourage password cycling
+    pw_min_age: 7
+    retries: 5
+    lockout_time: 600
+    timeout: 60
+    allow_homeless: False  # (type:boolean)
+    pam_passwdqc_enable: True  # (type:boolean)
+    pam_passwdqc_options: 'min=disabled,disabled,16,12,8'
+    root_ttys:
+        console
+        tty1
+        tty2
+        tty3
+        tty4
+        tty5
+        tty6
+    uid_min: 1000
+    gid_min: 1000
+    sys_uid_min: 100
+    sys_uid_max: 999
+    sys_gid_min: 100
+    sys_gid_max: 999
+    chfn_restrict:
+
+security:
+    users_allow: []
+    suid_sgid_enforce: True  # (type:boolean)
+    # user-defined blacklist and whitelist
+    suid_sgid_blacklist: []
+    suid_sgid_whitelist: []
+    # if this is True, remove any suid/sgid bits from files that were not in the whitelist
+    suid_sgid_dry_run_on_unknown: False  # (type:boolean)
+    suid_sgid_remove_from_unknown: False  # (type:boolean)
+    # remove packages with known issues
+    packages_clean: True  # (type:boolean)
+    packages_list:
+        xinetd
+        inetd
+        ypserv
+        telnet-server
+        rsh-server
+        rsync
+    kernel_enable_module_loading: True  # (type:boolean)
+    kernel_enable_core_dump: False  # (type:boolean)
+
+sysctl:
+    kernel_secure_sysrq: 244  # 4 + 16 + 32 + 64 + 128
+    kernel_enable_sysrq: False  # (type:boolean)
+    forwarding: False  # (type:boolean)
+    ipv6_enable: False  # (type:boolean)
+    arp_restricted: True  # (type:boolean)

hooks/charmhelpers/contrib/hardening/defaults/os.yaml.schema

@@ -0,0 +1,42 @@
+# NOTE: this schema must contain all valid keys from it's associated defaults
+#       file. It is used to validate user-provided overrides.
+general:
+    desktop_enable:
+environment:
+    extra_user_paths:
+    umask:
+    root_path:
+auth:
+    pw_max_age:
+    pw_min_age:
+    retries:
+    lockout_time:
+    timeout:
+    allow_homeless:
+    pam_passwdqc_enable:
+    pam_passwdqc_options:
+    root_ttys:
+    uid_min:
+    gid_min:
+    sys_uid_min:
+    sys_uid_max:
+    sys_gid_min:
+    sys_gid_max:
+    chfn_restrict:
+security:
+    users_allow:
+    suid_sgid_enforce:
+    suid_sgid_blacklist:
+    suid_sgid_whitelist:
+    suid_sgid_dry_run_on_unknown:
+    suid_sgid_remove_from_unknown:
+    packages_clean:
+    packages_list:
+    kernel_enable_module_loading:
+    kernel_enable_core_dump:
+sysctl:
+    kernel_secure_sysrq:
+    kernel_enable_sysrq:
+    forwarding:
+    ipv6_enable:
+    arp_restricted:

hooks/charmhelpers/contrib/hardening/defaults/ssh.yaml

@@ -0,0 +1,49 @@
+# NOTE: this file contains the default configuration for the 'ssh' hardening
+#       code. If you want to override any settings you must add them to a file
+#       called hardening.yaml in the root directory of your charm using the
+#       name 'ssh' as the root key followed by any of the following with new
+#       values.
+
+common:
+    service_name: 'ssh'
+    network_ipv6_enable: False  # (type:boolean)
+    ports: [22]
+    remote_hosts: []
+
+client:
+    package: 'openssh-client'
+    cbc_required: False  # (type:boolean)
+    weak_hmac: False  # (type:boolean)
+    weak_kex: False  # (type:boolean)
+    roaming: False
+    password_authentication: 'no'
+
+server:
+    host_key_files: ['/etc/ssh/ssh_host_rsa_key', '/etc/ssh/ssh_host_dsa_key',
+                     '/etc/ssh/ssh_host_ecdsa_key']
+    cbc_required: False  # (type:boolean)
+    weak_hmac: False  # (type:boolean)
+    weak_kex: False  # (type:boolean)
+    allow_root_with_key: False  # (type:boolean)
+    allow_tcp_forwarding: 'no'
+    allow_agent_forwarding: 'no'
+    allow_x11_forwarding: 'no'
+    use_privilege_separation: 'sandbox'
+    listen_to: ['0.0.0.0']
+    use_pam: 'no'
+    package: 'openssh-server'
+    password_authentication: 'no'
+    alive_interval: '600'
+    alive_count: '3'
+    sftp_enable: False  # (type:boolean)
+    sftp_group: 'sftponly'
+    sftp_chroot: '/home/%u'
+    deny_users: []
+    allow_users: []
+    deny_groups: []
+    allow_groups: []
+    print_motd: 'no'
+    print_last_log: 'no'
+    use_dns: 'no'
+    max_auth_tries: 2
+    max_sessions: 10

hooks/charmhelpers/contrib/hardening/defaults/ssh.yaml.schema

@@ -0,0 +1,42 @@
+# NOTE: this schema must contain all valid keys from it's associated defaults
+#       file. It is used to validate user-provided overrides.
+common:
+    service_name:
+    network_ipv6_enable:
+    ports:
+    remote_hosts:
+client:
+    package:
+    cbc_required:
+    weak_hmac:
+    weak_kex:
+    roaming:
+    password_authentication:
+server:
+    host_key_files:
+    cbc_required:
+    weak_hmac:
+    weak_kex:
+    allow_root_with_key:
+    allow_tcp_forwarding:
+    allow_agent_forwarding:
+    allow_x11_forwarding:
+    use_privilege_separation:
+    listen_to:
+    use_pam:
+    package:
+    password_authentication:
+    alive_interval:
+    alive_count:
+    sftp_enable:
+    sftp_group:
+    sftp_chroot:
+    deny_users:
+    allow_users:
+    deny_groups:
+    allow_groups:
+    print_motd:
+    print_last_log:
+    use_dns:
+    max_auth_tries:
+    max_sessions:

hooks/charmhelpers/contrib/hardening/harden.py

@@ -0,0 +1,84 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import six
+
+from collections import OrderedDict
+
+from charmhelpers.core.hookenv import (
+    config,
+    log,
+    DEBUG,
+    WARNING,
+)
+from charmhelpers.contrib.hardening.host.checks import run_os_checks
+from charmhelpers.contrib.hardening.ssh.checks import run_ssh_checks
+from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks
+from charmhelpers.contrib.hardening.apache.checks import run_apache_checks
+
+
+def harden(overrides=None):
+    """Hardening decorator.
+
+    This is the main entry point for running the hardening stack. In order to
+    run modules of the stack you must add this decorator to charm hook(s) and
+    ensure that your charm config.yaml contains the 'harden' option set to
+    one or more of the supported modules. Setting these will cause the
+    corresponding hardening code to be run when the hook fires.
+
+    This decorator can and should be applied to more than one hook or function
+    such that hardening modules are called multiple times. This is because
+    subsequent calls will perform auditing checks that will report any changes
+    to resources hardened by the first run (and possibly perform compliance
+    actions as a result of any detected infractions).
+
+    :param overrides: Optional list of stack modules used to override those
+                      provided with 'harden' config.
+    :returns: Returns value returned by decorated function once executed.
+    """
+    def _harden_inner1(f):
+        log("Hardening function '%s'" % (f.__name__), level=DEBUG)
+
+        def _harden_inner2(*args, **kwargs):
+            RUN_CATALOG = OrderedDict([('os', run_os_checks),
+                                       ('ssh', run_ssh_checks),
+                                       ('mysql', run_mysql_checks),
+                                       ('apache', run_apache_checks)])
+
+            enabled = overrides or (config("harden") or "").split()
+            if enabled:
+                modules_to_run = []
+                # modules will always be performed in the following order
+                for module, func in six.iteritems(RUN_CATALOG):
+                    if module in enabled:
+                        enabled.remove(module)
+                        modules_to_run.append(func)
+
+                if enabled:
+                    log("Unknown hardening modules '%s' - ignoring" %
+                        (', '.join(enabled)), level=WARNING)
+
+                for hardener in modules_to_run:
+                    log("Executing hardening module '%s'" %
+                        (hardener.__name__), level=DEBUG)
+                    hardener()
+            else:
+                log("No hardening applied to '%s'" % (f.__name__), level=DEBUG)
+
+            return f(*args, **kwargs)
+        return _harden_inner2
+
+    return _harden_inner1

hooks/charmhelpers/contrib/hardening/host/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from os import path
+
+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')

hooks/charmhelpers/contrib/hardening/host/checks/__init__.py

@@ -0,0 +1,50 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+)
+from charmhelpers.contrib.hardening.host.checks import (
+    apt,
+    limits,
+    login,
+    minimize_access,
+    pam,
+    profile,
+    securetty,
+    suid_sgid,
+    sysctl
+)
+
+
+def run_os_checks():
+    log("Starting OS hardening checks.", level=DEBUG)
+    checks = apt.get_audits()
+    checks.extend(limits.get_audits())
+    checks.extend(login.get_audits())
+    checks.extend(minimize_access.get_audits())
+    checks.extend(pam.get_audits())
+    checks.extend(profile.get_audits())
+    checks.extend(securetty.get_audits())
+    checks.extend(suid_sgid.get_audits())
+    checks.extend(sysctl.get_audits())
+
+    for check in checks:
+        log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
+        check.ensure_compliance()
+
+    log("OS hardening checks complete.", level=DEBUG)

hooks/charmhelpers/contrib/hardening/host/checks/apt.py

@@ -0,0 +1,39 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.contrib.hardening.utils import get_settings
+from charmhelpers.contrib.hardening.audits.apt import (
+    AptConfig,
+    RestrictedPackages,
+)
+
+
+def get_audits():
+    """Get OS hardening apt audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = [AptConfig([{'key': 'APT::Get::AllowUnauthenticated',
+                          'expected': 'false'}])]
+
+    settings = get_settings('os')
+    clean_packages = settings['security']['packages_clean']
+    if clean_packages:
+        security_packages = settings['security']['packages_list']
+        if security_packages:
+            audits.append(RestrictedPackages(security_packages))
+
+    return audits

hooks/charmhelpers/contrib/hardening/host/checks/limits.py

@@ -0,0 +1,55 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.contrib.hardening.audits.file import (
+    DirectoryPermissionAudit,
+    TemplatedFile,
+)
+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
+from charmhelpers.contrib.hardening import utils
+
+
+def get_audits():
+    """Get OS hardening security limits audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = []
+    settings = utils.get_settings('os')
+
+    # Ensure that the /etc/security/limits.d directory is only writable
+    # by the root user, but others can execute and read.
+    audits.append(DirectoryPermissionAudit('/etc/security/limits.d',
+                                           user='root', group='root',
+                                           mode=0o755))
+
+    # If core dumps are not enabled, then don't allow core dumps to be
+    # created as they may contain sensitive information.
+    if not settings['security']['kernel_enable_core_dump']:
+        audits.append(TemplatedFile('/etc/security/limits.d/10.hardcore.conf',
+                                    SecurityLimitsContext(),
+                                    template_dir=TEMPLATES_DIR,
+                                    user='root', group='root', mode=0o0440))
+    return audits
+
+
+class SecurityLimitsContext(object):
+
+    def __call__(self):
+        settings = utils.get_settings('os')
+        ctxt = {'disable_core_dump':
+                not settings['security']['kernel_enable_core_dump']}
+        return ctxt

hooks/charmhelpers/contrib/hardening/host/checks/login.py

@@ -0,0 +1,67 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from six import string_types
+
+from charmhelpers.contrib.hardening.audits.file import TemplatedFile
+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
+from charmhelpers.contrib.hardening import utils
+
+
+def get_audits():
+    """Get OS hardening login.defs audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = [TemplatedFile('/etc/login.defs', LoginContext(),
+                            template_dir=TEMPLATES_DIR,
+                            user='root', group='root', mode=0o0444)]
+    return audits
+
+
+class LoginContext(object):
+
+    def __call__(self):
+        settings = utils.get_settings('os')
+
+        # Octal numbers in yaml end up being turned into decimal,
+        # so check if the umask is entered as a string (e.g. '027')
+        # or as an octal umask as we know it (e.g. 002). If its not
+        # a string assume it to be octal and turn it into an octal
+        # string.
+        umask = settings['environment']['umask']
+        if not isinstance(umask, string_types):
+            umask = '%s' % oct(umask)
+
+        ctxt = {
+            'additional_user_paths':
+            settings['environment']['extra_user_paths'],
+            'umask': umask,
+            'pwd_max_age': settings['auth']['pw_max_age'],
+            'pwd_min_age': settings['auth']['pw_min_age'],
+            'uid_min': settings['auth']['uid_min'],
+            'sys_uid_min': settings['auth']['sys_uid_min'],
+            'sys_uid_max': settings['auth']['sys_uid_max'],
+            'gid_min': settings['auth']['gid_min'],
+            'sys_gid_min': settings['auth']['sys_gid_min'],
+            'sys_gid_max': settings['auth']['sys_gid_max'],
+            'login_retries': settings['auth']['retries'],
+            'login_timeout': settings['auth']['timeout'],
+            'chfn_restrict': settings['auth']['chfn_restrict'],
+            'allow_login_without_home': settings['auth']['allow_homeless']
+        }
+
+        return ctxt

hooks/charmhelpers/contrib/hardening/host/checks/minimize_access.py

@@ -0,0 +1,52 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.contrib.hardening.audits.file import (
+    FilePermissionAudit,
+    ReadOnly,
+)
+from charmhelpers.contrib.hardening import utils
+
+
+def get_audits():
+    """Get OS hardening access audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = []
+    settings = utils.get_settings('os')
+
+    # Remove write permissions from $PATH folders for all regular users.
+    # This prevents changing system-wide commands from normal users.
+    path_folders = {'/usr/local/sbin',
+                    '/usr/local/bin',
+                    '/usr/sbin',
+                    '/usr/bin',
+                    '/bin'}
+    extra_user_paths = settings['environment']['extra_user_paths']
+    path_folders.update(extra_user_paths)
+    audits.append(ReadOnly(path_folders))
+
+    # Only allow the root user to have access to the shadow file.
+    audits.append(FilePermissionAudit('/etc/shadow', 'root', 'root', 0o0600))
+
+    if 'change_user' not in settings['security']['users_allow']:
+        # su should only be accessible to user and group root, unless it is
+        # expressly defined to allow users to change to root via the
+        # security_users_allow config option.
+        audits.append(FilePermissionAudit('/bin/su', 'root', 'root', 0o750))
+
+    return audits

hooks/charmhelpers/contrib/hardening/host/checks/pam.py

@@ -0,0 +1,134 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from subprocess import (
+    check_output,
+    CalledProcessError,
+)
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+    ERROR,
+)
+from charmhelpers.fetch import (
+    apt_install,
+    apt_purge,
+    apt_update,
+)
+from charmhelpers.contrib.hardening.audits.file import (
+    TemplatedFile,
+    DeletedFile,
+)
+from charmhelpers.contrib.hardening import utils
+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
+
+
+def get_audits():
+    """Get OS hardening PAM authentication audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = []
+
+    settings = utils.get_settings('os')
+
+    if settings['auth']['pam_passwdqc_enable']:
+        audits.append(PasswdqcPAM('/etc/passwdqc.conf'))
+
+    if settings['auth']['retries']:
+        audits.append(Tally2PAM('/usr/share/pam-configs/tally2'))
+    else:
+        audits.append(DeletedFile('/usr/share/pam-configs/tally2'))
+
+    return audits
+
+
+class PasswdqcPAMContext(object):
+
+    def __call__(self):
+        ctxt = {}
+        settings = utils.get_settings('os')
+
+        ctxt['auth_pam_passwdqc_options'] = \
+            settings['auth']['pam_passwdqc_options']
+
+        return ctxt
+
+
+class PasswdqcPAM(TemplatedFile):
+    """The PAM Audit verifies the linux PAM settings."""
+    def __init__(self, path):
+        super(PasswdqcPAM, self).__init__(path=path,
+                                          template_dir=TEMPLATES_DIR,
+                                          context=PasswdqcPAMContext(),
+                                          user='root',
+                                          group='root',
+                                          mode=0o0640)
+
+    def pre_write(self):
+        # Always remove?
+        for pkg in ['libpam-ccreds', 'libpam-cracklib']:
+            log("Purging package '%s'" % pkg, level=DEBUG),
+            apt_purge(pkg)
+
+        apt_update(fatal=True)
+        for pkg in ['libpam-passwdqc']:
+            log("Installing package '%s'" % pkg, level=DEBUG),
+            apt_install(pkg)
+
+    def post_write(self):
+        """Updates the PAM configuration after the file has been written"""
+        try:
+            check_output(['pam-auth-update', '--package'])
+        except CalledProcessError as e:
+            log('Error calling pam-auth-update: %s' % e, level=ERROR)
+
+
+class Tally2PAMContext(object):
+
+    def __call__(self):
+        ctxt = {}
+        settings = utils.get_settings('os')
+
+        ctxt['auth_lockout_time'] = settings['auth']['lockout_time']
+        ctxt['auth_retries'] = settings['auth']['retries']
+
+        return ctxt
+
+
+class Tally2PAM(TemplatedFile):
+    """The PAM Audit verifies the linux PAM settings."""
+    def __init__(self, path):
+        super(Tally2PAM, self).__init__(path=path,
+                                        template_dir=TEMPLATES_DIR,
+                                        context=Tally2PAMContext(),
+                                        user='root',
+                                        group='root',
+                                        mode=0o0640)
+
+    def pre_write(self):
+        # Always remove?
+        apt_purge('libpam-ccreds')
+        apt_update(fatal=True)
+        apt_install('libpam-modules')
+
+    def post_write(self):
+        """Updates the PAM configuration after the file has been written"""
+        try:
+            check_output(['pam-auth-update', '--package'])
+        except CalledProcessError as e:
+            log('Error calling pam-auth-update: %s' % e, level=ERROR)

hooks/charmhelpers/contrib/hardening/host/checks/profile.py

@@ -0,0 +1,45 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.contrib.hardening.audits.file import TemplatedFile
+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
+from charmhelpers.contrib.hardening import utils
+
+
+def get_audits():
+    """Get OS hardening profile audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = []
+
+    settings = utils.get_settings('os')
+
+    # If core dumps are not enabled, then don't allow core dumps to be
+    # created as they may contain sensitive information.
+    if not settings['security']['kernel_enable_core_dump']:
+        audits.append(TemplatedFile('/etc/profile.d/pinerolo_profile.sh',
+                                    ProfileContext(),
+                                    template_dir=TEMPLATES_DIR,
+                                    mode=0o0755, user='root', group='root'))
+    return audits
+
+
+class ProfileContext(object):
+
+    def __call__(self):
+        ctxt = {}
+        return ctxt

hooks/charmhelpers/contrib/hardening/host/checks/securetty.py

@@ -0,0 +1,39 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.contrib.hardening.audits.file import TemplatedFile
+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
+from charmhelpers.contrib.hardening import utils
+
+
+def get_audits():
+    """Get OS hardening Secure TTY audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = []
+    audits.append(TemplatedFile('/etc/securetty', SecureTTYContext(),
+                                template_dir=TEMPLATES_DIR,
+                                mode=0o0400, user='root', group='root'))
+    return audits
+
+
+class SecureTTYContext(object):
+
+    def __call__(self):
+        settings = utils.get_settings('os')
+        ctxt = {'ttys': settings['auth']['root_ttys']}
+        return ctxt

hooks/charmhelpers/contrib/hardening/host/checks/suid_sgid.py

@@ -0,0 +1,131 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import subprocess
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO,
+)
+from charmhelpers.contrib.hardening.audits.file import NoSUIDSGIDAudit
+from charmhelpers.contrib.hardening import utils
+
+
+BLACKLIST = ['/usr/bin/rcp', '/usr/bin/rlogin', '/usr/bin/rsh',
+             '/usr/libexec/openssh/ssh-keysign',
+             '/usr/lib/openssh/ssh-keysign',
+             '/sbin/netreport',
+             '/usr/sbin/usernetctl',
+             '/usr/sbin/userisdnctl',
+             '/usr/sbin/pppd',
+             '/usr/bin/lockfile',
+             '/usr/bin/mail-lock',
+             '/usr/bin/mail-unlock',
+             '/usr/bin/mail-touchlock',
+             '/usr/bin/dotlockfile',
+             '/usr/bin/arping',
+             '/usr/sbin/uuidd',
+             '/usr/bin/mtr',
+             '/usr/lib/evolution/camel-lock-helper-1.2',
+             '/usr/lib/pt_chown',
+             '/usr/lib/eject/dmcrypt-get-device',
+             '/usr/lib/mc/cons.saver']
+
+WHITELIST = ['/bin/mount', '/bin/ping', '/bin/su', '/bin/umount',
+             '/sbin/pam_timestamp_check', '/sbin/unix_chkpwd', '/usr/bin/at',
+             '/usr/bin/gpasswd', '/usr/bin/locate', '/usr/bin/newgrp',
+             '/usr/bin/passwd', '/usr/bin/ssh-agent',
+             '/usr/libexec/utempter/utempter', '/usr/sbin/lockdev',
+             '/usr/sbin/sendmail.sendmail', '/usr/bin/expiry',
+             '/bin/ping6', '/usr/bin/traceroute6.iputils',
+             '/sbin/mount.nfs', '/sbin/umount.nfs',
+             '/sbin/mount.nfs4', '/sbin/umount.nfs4',
+             '/usr/bin/crontab',
+             '/usr/bin/wall', '/usr/bin/write',
+             '/usr/bin/screen',
+             '/usr/bin/mlocate',
+             '/usr/bin/chage', '/usr/bin/chfn', '/usr/bin/chsh',
+             '/bin/fusermount',
+             '/usr/bin/pkexec',
+             '/usr/bin/sudo', '/usr/bin/sudoedit',
+             '/usr/sbin/postdrop', '/usr/sbin/postqueue',
+             '/usr/sbin/suexec',
+             '/usr/lib/squid/ncsa_auth', '/usr/lib/squid/pam_auth',
+             '/usr/kerberos/bin/ksu',
+             '/usr/sbin/ccreds_validate',
+             '/usr/bin/Xorg',
+             '/usr/bin/X',
+             '/usr/lib/dbus-1.0/dbus-daemon-launch-helper',
+             '/usr/lib/vte/gnome-pty-helper',
+             '/usr/lib/libvte9/gnome-pty-helper',
+             '/usr/lib/libvte-2.90-9/gnome-pty-helper']
+
+
+def get_audits():
+    """Get OS hardening suid/sgid audits.
+
+    :returns:  dictionary of audits
+    """
+    checks = []
+    settings = utils.get_settings('os')
+    if not settings['security']['suid_sgid_enforce']:
+        log("Skipping suid/sgid hardening", level=INFO)
+        return checks
+
+    # Build the blacklist and whitelist of files for suid/sgid checks.
+    # There are a total of 4 lists:
+    #   1. the system blacklist
+    #   2. the system whitelist
+    #   3. the user blacklist
+    #   4. the user whitelist
+    #
+    # The blacklist is the set of paths which should NOT have the suid/sgid bit
+    # set and the whitelist is the set of paths which MAY have the suid/sgid
+    # bit setl. The user whitelist/blacklist effectively override the system
+    # whitelist/blacklist.
+    u_b = settings['security']['suid_sgid_blacklist']
+    u_w = settings['security']['suid_sgid_whitelist']
+
+    blacklist = set(BLACKLIST) - set(u_w + u_b)
+    whitelist = set(WHITELIST) - set(u_b + u_w)
+
+    checks.append(NoSUIDSGIDAudit(blacklist))
+
+    dry_run = settings['security']['suid_sgid_dry_run_on_unknown']
+
+    if settings['security']['suid_sgid_remove_from_unknown'] or dry_run:
+        # If the policy is a dry_run (e.g. complain only) or remove unknown
+        # suid/sgid bits then find all of the paths which have the suid/sgid
+        # bit set and then remove the whitelisted paths.
+        root_path = settings['environment']['root_path']
+        unknown_paths = find_paths_with_suid_sgid(root_path) - set(whitelist)
+        checks.append(NoSUIDSGIDAudit(unknown_paths, unless=dry_run))
+
+    return checks
+
+
+def find_paths_with_suid_sgid(root_path):
+    """Finds all paths/files which have an suid/sgid bit enabled.
+
+    Starting with the root_path, this will recursively find all paths which
+    have an suid or sgid bit set.
+    """
+    cmd = ['find', root_path, '-perm', '-4000', '-o', '-perm', '-2000',
+           '-type', 'f', '!', '-path', '/proc/*', '-print']
+
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, _ = p.communicate()
+    return set(out.split('\n'))

hooks/charmhelpers/contrib/hardening/host/checks/sysctl.py

@@ -0,0 +1,211 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import platform
+import re
+import six
+import subprocess
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO,
+    WARNING,
+)
+from charmhelpers.contrib.hardening import utils
+from charmhelpers.contrib.hardening.audits.file import (
+    FilePermissionAudit,
+    TemplatedFile,
+)
+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
+
+
+SYSCTL_DEFAULTS = """net.ipv4.ip_forward=%(net_ipv4_ip_forward)s
+net.ipv6.conf.all.forwarding=%(net_ipv6_conf_all_forwarding)s
+net.ipv4.conf.all.rp_filter=1
+net.ipv4.conf.default.rp_filter=1
+net.ipv4.icmp_echo_ignore_broadcasts=1
+net.ipv4.icmp_ignore_bogus_error_responses=1
+net.ipv4.icmp_ratelimit=100
+net.ipv4.icmp_ratemask=88089
+net.ipv6.conf.all.disable_ipv6=%(net_ipv6_conf_all_disable_ipv6)s
+net.ipv4.tcp_timestamps=%(net_ipv4_tcp_timestamps)s
+net.ipv4.conf.all.arp_ignore=%(net_ipv4_conf_all_arp_ignore)s
+net.ipv4.conf.all.arp_announce=%(net_ipv4_conf_all_arp_announce)s
+net.ipv4.tcp_rfc1337=1
+net.ipv4.tcp_syncookies=1
+net.ipv4.conf.all.shared_media=1
+net.ipv4.conf.default.shared_media=1
+net.ipv4.conf.all.accept_source_route=0
+net.ipv4.conf.default.accept_source_route=0
+net.ipv4.conf.all.accept_redirects=0
+net.ipv4.conf.default.accept_redirects=0
+net.ipv6.conf.all.accept_redirects=0
+net.ipv6.conf.default.accept_redirects=0
+net.ipv4.conf.all.secure_redirects=0
+net.ipv4.conf.default.secure_redirects=0
+net.ipv4.conf.all.send_redirects=0
+net.ipv4.conf.default.send_redirects=0
+net.ipv4.conf.all.log_martians=0
+net.ipv6.conf.default.router_solicitations=0
+net.ipv6.conf.default.accept_ra_rtr_pref=0
+net.ipv6.conf.default.accept_ra_pinfo=0
+net.ipv6.conf.default.accept_ra_defrtr=0
+net.ipv6.conf.default.autoconf=0
+net.ipv6.conf.default.dad_transmits=0
+net.ipv6.conf.default.max_addresses=1
+net.ipv6.conf.all.accept_ra=0
+net.ipv6.conf.default.accept_ra=0
+kernel.modules_disabled=%(kernel_modules_disabled)s
+kernel.sysrq=%(kernel_sysrq)s
+fs.suid_dumpable=%(fs_suid_dumpable)s
+kernel.randomize_va_space=2
+"""
+
+
+def get_audits():
+    """Get OS hardening sysctl audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = []
+    settings = utils.get_settings('os')
+
+    # Apply the sysctl settings which are configured to be applied.
+    audits.append(SysctlConf())
+    # Make sure that only root has access to the sysctl.conf file, and
+    # that it is read-only.
+    audits.append(FilePermissionAudit('/etc/sysctl.conf',
+                                      user='root',
+                                      group='root', mode=0o0440))
+    # If module loading is not enabled, then ensure that the modules
+    # file has the appropriate permissions and rebuild the initramfs
+    if not settings['security']['kernel_enable_module_loading']:
+        audits.append(ModulesTemplate())
+
+    return audits
+
+
+class ModulesContext(object):
+
+    def __call__(self):
+        settings = utils.get_settings('os')
+        with open('/proc/cpuinfo', 'r') as fd:
+            cpuinfo = fd.readlines()
+
+        for line in cpuinfo:
+            match = re.search(r"^vendor_id\s+:\s+(.+)", line)
+            if match:
+                vendor = match.group(1)
+
+        if vendor == "GenuineIntel":
+            vendor = "intel"
+        elif vendor == "AuthenticAMD":
+            vendor = "amd"
+
+        ctxt = {'arch': platform.processor(),
+                'cpuVendor': vendor,
+                'desktop_enable': settings['general']['desktop_enable']}
+
+        return ctxt
+
+
+class ModulesTemplate(object):
+
+    def __init__(self):
+        super(ModulesTemplate, self).__init__('/etc/initramfs-tools/modules',
+                                              ModulesContext(),
+                                              templates_dir=TEMPLATES_DIR,
+                                              user='root', group='root',
+                                              mode=0o0440)
+
+    def post_write(self):
+        subprocess.check_call(['update-initramfs', '-u'])
+
+
+class SysCtlHardeningContext(object):
+    def __call__(self):
+        settings = utils.get_settings('os')
+        ctxt = {'sysctl': {}}
+
+        log("Applying sysctl settings", level=INFO)
+        extras = {'net_ipv4_ip_forward': 0,
+                  'net_ipv6_conf_all_forwarding': 0,
+                  'net_ipv6_conf_all_disable_ipv6': 1,
+                  'net_ipv4_tcp_timestamps': 0,
+                  'net_ipv4_conf_all_arp_ignore': 0,
+                  'net_ipv4_conf_all_arp_announce': 0,
+                  'kernel_sysrq': 0,
+                  'fs_suid_dumpable': 0,
+                  'kernel_modules_disabled': 1}
+
+        if settings['sysctl']['ipv6_enable']:
+            extras['net_ipv6_conf_all_disable_ipv6'] = 0
+
+        if settings['sysctl']['forwarding']:
+            extras['net_ipv4_ip_forward'] = 1
+            extras['net_ipv6_conf_all_forwarding'] = 1
+
+        if settings['sysctl']['arp_restricted']:
+            extras['net_ipv4_conf_all_arp_ignore'] = 1
+            extras['net_ipv4_conf_all_arp_announce'] = 2
+
+        if settings['security']['kernel_enable_module_loading']:
+            extras['kernel_modules_disabled'] = 0
+
+        if settings['sysctl']['kernel_enable_sysrq']:
+            sysrq_val = settings['sysctl']['kernel_secure_sysrq']
+            extras['kernel_sysrq'] = sysrq_val
+
+        if settings['security']['kernel_enable_core_dump']:
+            extras['fs_suid_dumpable'] = 1
+
+        settings.update(extras)
+        for d in (SYSCTL_DEFAULTS % settings).split():
+            d = d.strip().partition('=')
+            key = d[0].strip()
+            path = os.path.join('/proc/sys', key.replace('.', '/'))
+            if not os.path.exists(path):
+                log("Skipping '%s' since '%s' does not exist" % (key, path),
+                    level=WARNING)
+                continue
+
+            ctxt['sysctl'][key] = d[2] or None
+
+        # Translate for python3
+        return {'sysctl_settings':
+                [(k, v) for k, v in six.iteritems(ctxt['sysctl'])]}
+
+
+class SysctlConf(TemplatedFile):
+    """An audit check for sysctl settings."""
+    def __init__(self):
+        self.conffile = '/etc/sysctl.d/99-juju-hardening.conf'
+        super(SysctlConf, self).__init__(self.conffile,
+                                         SysCtlHardeningContext(),
+                                         template_dir=TEMPLATES_DIR,
+                                         user='root', group='root',
+                                         mode=0o0440)
+
+    def post_write(self):
+        try:
+            subprocess.check_call(['sysctl', '-p', self.conffile])
+        except subprocess.CalledProcessError as e:
+            # NOTE: on some systems if sysctl cannot apply all settings it
+            #       will return non-zero as well.
+            log("sysctl command returned an error (maybe some "
+                "keys could not be set) - %s" % (e),
+                level=WARNING)

hooks/charmhelpers/contrib/hardening/host/templates/10.hardcore.conf

@@ -0,0 +1,8 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+{% if disable_core_dump -%}
+# Prevent core dumps for all users. These are usually only needed by developers and may contain sensitive information.
+* hard core 0
+{% endif %}

hooks/charmhelpers/contrib/hardening/host/templates/99-juju-hardening.conf

@@ -0,0 +1,7 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+{% for key, value in sysctl_settings -%}
+{{ key }}={{ value }}
+{% endfor -%}

hooks/charmhelpers/contrib/hardening/host/templates/login.defs

@@ -0,0 +1,349 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+#
+# /etc/login.defs - Configuration control definitions for the login package.
+#
+# Three items must be defined:  MAIL_DIR, ENV_SUPATH, and ENV_PATH.
+# If unspecified, some arbitrary (and possibly incorrect) value will
+# be assumed.  All other items are optional - if not specified then
+# the described action or option will be inhibited.
+#
+# Comment lines (lines beginning with "#") and blank lines are ignored.
+#
+# Modified for Linux.  --marekm
+
+# REQUIRED for useradd/userdel/usermod
+#   Directory where mailboxes reside, _or_ name of file, relative to the
+#   home directory.  If you _do_ define MAIL_DIR and MAIL_FILE,
+#   MAIL_DIR takes precedence.
+#
+#   Essentially:
+#      - MAIL_DIR defines the location of users mail spool files
+#        (for mbox use) by appending the username to MAIL_DIR as defined
+#        below.
+#      - MAIL_FILE defines the location of the users mail spool files as the
+#        fully-qualified filename obtained by prepending the user home
+#        directory before $MAIL_FILE
+#
+# NOTE: This is no more used for setting up users MAIL environment variable
+#       which is, starting from shadow 4.0.12-1 in Debian, entirely the
+#       job of the pam_mail PAM modules
+#       See default PAM configuration files provided for
+#       login, su, etc.
+#
+# This is a temporary situation: setting these variables will soon
+# move to /etc/default/useradd and the variables will then be
+# no more supported
+MAIL_DIR        /var/mail
+#MAIL_FILE      .mail
+
+#
+# Enable logging and display of /var/log/faillog login failure info.
+# This option conflicts with the pam_tally PAM module.
+#
+FAILLOG_ENAB		yes
+
+#
+# Enable display of unknown usernames when login failures are recorded.
+#
+# WARNING: Unknown usernames may become world readable. 
+# See #290803 and #298773 for details about how this could become a security
+# concern
+LOG_UNKFAIL_ENAB	no
+
+#
+# Enable logging of successful logins
+#
+LOG_OK_LOGINS		yes
+
+#
+# Enable "syslog" logging of su activity - in addition to sulog file logging.
+# SYSLOG_SG_ENAB does the same for newgrp and sg.
+#
+SYSLOG_SU_ENAB		yes
+SYSLOG_SG_ENAB		yes
+
+#
+# If defined, all su activity is logged to this file.
+#
+#SULOG_FILE	/var/log/sulog
+
+#
+# If defined, file which maps tty line to TERM environment parameter.
+# Each line of the file is in a format something like "vt100  tty01".
+#
+#TTYTYPE_FILE	/etc/ttytype
+
+#
+# If defined, login failures will be logged here in a utmp format
+# last, when invoked as lastb, will read /var/log/btmp, so...
+#
+FTMP_FILE	/var/log/btmp
+
+#
+# If defined, the command name to display when running "su -".  For
+# example, if this is defined as "su" then a "ps" will display the
+# command is "-su".  If not defined, then "ps" would display the
+# name of the shell actually being run, e.g. something like "-sh".
+#
+SU_NAME		su
+
+#
+# If defined, file which inhibits all the usual chatter during the login
+# sequence.  If a full pathname, then hushed mode will be enabled if the
+# user's name or shell are found in the file.  If not a full pathname, then
+# hushed mode will be enabled if the file exists in the user's home directory.
+#
+HUSHLOGIN_FILE	.hushlogin
+#HUSHLOGIN_FILE	/etc/hushlogins
+
+#
+# *REQUIRED*  The default PATH settings, for superuser and normal users.
+#
+# (they are minimal, add the rest in the shell startup files)
+ENV_SUPATH	PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV_PATH	PATH=/usr/local/bin:/usr/bin:/bin{% if additional_user_paths %}{{ additional_user_paths }}{% endif %}
+
+#
+# Terminal permissions
+#
+#	TTYGROUP	Login tty will be assigned this group ownership.
+#	TTYPERM		Login tty will be set to this permission.
+#
+# If you have a "write" program which is "setgid" to a special group
+# which owns the terminals, define TTYGROUP to the group number and
+# TTYPERM to 0620.  Otherwise leave TTYGROUP commented out and assign
+# TTYPERM to either 622 or 600.
+#
+# In Debian /usr/bin/bsd-write or similar programs are setgid tty
+# However, the default and recommended value for TTYPERM is still 0600
+# to not allow anyone to write to anyone else console or terminal
+
+# Users can still allow other people to write them by issuing 
+# the "mesg y" command.
+
+TTYGROUP	tty
+TTYPERM		0600
+
+#
+# Login configuration initializations:
+#
+#	ERASECHAR	Terminal ERASE character ('\010' = backspace).
+#	KILLCHAR	Terminal KILL character ('\025' = CTRL/U).
+#	UMASK		Default "umask" value.
+#
+# The ERASECHAR and KILLCHAR are used only on System V machines.
+# 
+# UMASK is the default umask value for pam_umask and is used by
+# useradd and newusers to set the mode of the new home directories.
+# 022 is the "historical" value in Debian for UMASK
+# 027, or even 077, could be considered better for privacy
+# There is no One True Answer here : each sysadmin must make up his/her
+# mind.
+#
+# If USERGROUPS_ENAB is set to "yes", that will modify this UMASK default value
+# for private user groups, i. e. the uid is the same as gid, and username is
+# the same as the primary group name: for these, the user permissions will be
+# used as group permissions, e. g. 022 will become 002.
+#
+# Prefix these values with "0" to get octal, "0x" to get hexadecimal.
+#
+ERASECHAR	0177
+KILLCHAR	025
+UMASK		{{ umask }}
+
+# Enable setting of the umask group bits to be the same as owner bits (examples: `022` -> `002`, `077` -> `007`) for non-root users, if the uid is the same as gid, and username is the same as the primary group name.
+# If set to yes, userdel will remove the user´s group if it contains no more members, and useradd will create by default a group with the name of the user.
+USERGROUPS_ENAB yes
+
+#
+# Password aging controls:
+#
+#	PASS_MAX_DAYS	Maximum number of days a password may be used.
+#	PASS_MIN_DAYS	Minimum number of days allowed between password changes.
+#	PASS_WARN_AGE	Number of days warning given before a password expires.
+#
+PASS_MAX_DAYS	{{ pwd_max_age }}
+PASS_MIN_DAYS	{{ pwd_min_age }}
+PASS_WARN_AGE	7
+
+#
+# Min/max values for automatic uid selection in useradd
+#
+UID_MIN			 {{ uid_min }}
+UID_MAX			60000
+# System accounts
+SYS_UID_MIN		  {{ sys_uid_min }}
+SYS_UID_MAX		  {{ sys_uid_max }}
+
+# Min/max values for automatic gid selection in groupadd
+GID_MIN			 {{ gid_min }}
+GID_MAX			60000
+# System accounts
+SYS_GID_MIN		  {{ sys_gid_min }}
+SYS_GID_MAX		  {{ sys_gid_max }}
+
+#
+# Max number of login retries if password is bad. This will most likely be
+# overriden by PAM, since the default pam_unix module has it's own built
+# in of 3 retries. However, this is a safe fallback in case you are using
+# an authentication module that does not enforce PAM_MAXTRIES.
+#
+LOGIN_RETRIES		{{ login_retries }}
+
+#
+# Max time in seconds for login
+#
+LOGIN_TIMEOUT		{{ login_timeout }}
+
+#
+# Which fields may be changed by regular users using chfn - use
+# any combination of letters "frwh" (full name, room number, work
+# phone, home phone).  If not defined, no changes are allowed.
+# For backward compatibility, "yes" = "rwh" and "no" = "frwh".
+# 
+{% if chfn_restrict %}
+CHFN_RESTRICT		{{ chfn_restrict }}
+{% endif %}
+
+#
+# Should login be allowed if we can't cd to the home directory?
+# Default in no.
+#
+DEFAULT_HOME	{% if allow_login_without_home %} yes {% else %} no {% endif %}
+
+#
+# If defined, this command is run when removing a user.
+# It should remove any at/cron/print jobs etc. owned by
+# the user to be removed (passed as the first argument).
+#
+#USERDEL_CMD	/usr/sbin/userdel_local
+
+#
+# Enable setting of the umask group bits to be the same as owner bits
+# (examples: 022 -> 002, 077 -> 007) for non-root users, if the uid is
+# the same as gid, and username is the same as the primary group name.
+#
+# If set to yes, userdel will remove the user´s group if it contains no
+# more members, and useradd will create by default a group with the name
+# of the user.
+#
+USERGROUPS_ENAB yes
+
+#
+# Instead of the real user shell, the program specified by this parameter
+# will be launched, although its visible name (argv[0]) will be the shell's.
+# The program may do whatever it wants (logging, additional authentification,
+# banner, ...) before running the actual shell.
+#
+# FAKE_SHELL /bin/fakeshell
+
+#
+# If defined, either full pathname of a file containing device names or
+# a ":" delimited list of device names.  Root logins will be allowed only
+# upon these devices.
+#
+# This variable is used by login and su.
+#
+#CONSOLE	/etc/consoles
+#CONSOLE	console:tty01:tty02:tty03:tty04
+
+#
+# List of groups to add to the user's supplementary group set
+# when logging in on the console (as determined by the CONSOLE
+# setting).  Default is none.
+#
+# Use with caution - it is possible for users to gain permanent
+# access to these groups, even when not logged in on the console.
+# How to do it is left as an exercise for the reader...
+#
+# This variable is used by login and su.
+#
+#CONSOLE_GROUPS		floppy:audio:cdrom
+
+#
+# If set to "yes", new passwords will be encrypted using the MD5-based
+# algorithm compatible with the one used by recent releases of FreeBSD.
+# It supports passwords of unlimited length and longer salt strings.
+# Set to "no" if you need to copy encrypted passwords to other systems
+# which don't understand the new algorithm.  Default is "no".
+#
+# This variable is deprecated. You should use ENCRYPT_METHOD.
+#
+MD5_CRYPT_ENAB	no
+
+#
+# If set to MD5 , MD5-based algorithm will be used for encrypting password
+# If set to SHA256, SHA256-based algorithm will be used for encrypting password
+# If set to SHA512, SHA512-based algorithm will be used for encrypting password
+# If set to DES, DES-based algorithm will be used for encrypting password (default)
+# Overrides the MD5_CRYPT_ENAB option
+#
+# Note: It is recommended to use a value consistent with
+# the PAM modules configuration.
+#
+ENCRYPT_METHOD SHA512
+
+#
+# Only used if ENCRYPT_METHOD is set to SHA256 or SHA512.
+#
+# Define the number of SHA rounds.
+# With a lot of rounds, it is more difficult to brute forcing the password.
+# But note also that it more CPU resources will be needed to authenticate
+# users.
+#
+# If not specified, the libc will choose the default number of rounds (5000).
+# The values must be inside the 1000-999999999 range.
+# If only one of the MIN or MAX values is set, then this value will be used.
+# If MIN > MAX, the highest value will be used.
+#
+# SHA_CRYPT_MIN_ROUNDS 5000
+# SHA_CRYPT_MAX_ROUNDS 5000
+
+################# OBSOLETED BY PAM ##############
+#						#
+# These options are now handled by PAM. Please	#
+# edit the appropriate file in /etc/pam.d/ to	#
+# enable the equivelants of them.
+#
+###############
+
+#MOTD_FILE
+#DIALUPS_CHECK_ENAB
+#LASTLOG_ENAB
+#MAIL_CHECK_ENAB
+#OBSCURE_CHECKS_ENAB
+#PORTTIME_CHECKS_ENAB
+#SU_WHEEL_ONLY
+#CRACKLIB_DICTPATH
+#PASS_CHANGE_TRIES
+#PASS_ALWAYS_WARN
+#ENVIRON_FILE
+#NOLOGINS_FILE
+#ISSUE_FILE
+#PASS_MIN_LEN
+#PASS_MAX_LEN
+#ULIMIT
+#ENV_HZ
+#CHFN_AUTH
+#CHSH_AUTH
+#FAIL_DELAY
+
+################# OBSOLETED #######################
+#						  #
+# These options are no more handled by shadow.    #
+#                                                 #
+# Shadow utilities will display a warning if they #
+# still appear.                                   #
+#                                                 #
+###################################################
+
+# CLOSE_SESSIONS
+# LOGIN_STRING
+# NO_PASSWORD_CONSOLE
+# QMAIL_DIR
+
+
+

hooks/charmhelpers/contrib/hardening/host/templates/modules

@@ -0,0 +1,117 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+# /etc/modules: kernel modules to load at boot time.
+#
+# This file contains the names of kernel modules that should be loaded
+# at boot time, one per line. Lines beginning with "#" are ignored.
+# Parameters can be specified after the module name.
+
+# Arch
+# ----
+# 
+# Modules for certains builds, contains support modules and some CPU-specific optimizations.
+
+{% if arch == "x86_64" -%}
+# Optimize for x86_64 cryptographic features
+twofish-x86_64-3way
+twofish-x86_64
+aes-x86_64
+salsa20-x86_64
+blowfish-x86_64
+{% endif -%}
+
+{% if cpuVendor == "intel" -%}
+# Intel-specific optimizations
+ghash-clmulni-intel
+aesni-intel
+kvm-intel
+{% endif -%}
+
+{% if cpuVendor == "amd" -%}
+# AMD-specific optimizations
+kvm-amd
+{% endif -%}
+
+kvm
+
+
+# Crypto
+# ------
+
+# Some core modules which comprise strong cryptography.
+blowfish_common
+blowfish_generic
+ctr
+cts
+lrw
+lzo
+rmd160
+rmd256
+rmd320
+serpent
+sha512_generic
+twofish_common
+twofish_generic
+xts
+zlib
+
+
+# Drivers
+# -------
+
+# Basics
+lp
+rtc
+loop
+
+# Filesystems
+ext2
+btrfs
+
+{% if desktop_enable -%}
+# Desktop
+psmouse
+snd
+snd_ac97_codec
+snd_intel8x0
+snd_page_alloc
+snd_pcm
+snd_timer
+soundcore
+usbhid
+{% endif -%}
+
+# Lib
+# ---
+xz
+
+
+# Net
+# ---
+
+# All packets needed for netfilter rules (ie iptables, ebtables).
+ip_tables
+x_tables
+iptable_filter
+iptable_nat
+
+# Targets
+ipt_LOG
+ipt_REJECT
+
+# Modules
+xt_connlimit
+xt_tcpudp
+xt_recent
+xt_limit
+xt_conntrack
+nf_conntrack
+nf_conntrack_ipv4
+nf_defrag_ipv4
+xt_state
+nf_nat
+
+# Addons
+xt_pknock

hooks/charmhelpers/contrib/hardening/host/templates/passwdqc.conf

@@ -0,0 +1,11 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+Name: passwdqc password strength enforcement
+Default: yes
+Priority: 1024
+Conflicts: cracklib
+Password-Type: Primary
+Password:
+  requisite     pam_passwdqc.so {{ auth_pam_passwdqc_options }}

hooks/charmhelpers/contrib/hardening/host/templates/pinerolo_profile.sh

@@ -0,0 +1,8 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+# Disable core dumps via soft limits for all users. Compliance to this setting
+# is voluntary and can be modified by users up to a hard limit. This setting is
+# a sane default.
+ulimit -S -c 0 > /dev/null 2>&1

hooks/charmhelpers/contrib/hardening/host/templates/securetty

@@ -0,0 +1,11 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+# A list of TTYs, from which root can log in
+# see `man securetty` for reference
+{% if ttys -%}
+{% for tty in ttys -%}
+{{ tty }}
+{% endfor -%}
+{% endif -%}

hooks/charmhelpers/contrib/hardening/host/templates/tally2

@@ -0,0 +1,14 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+Name: tally2 lockout after failed attempts enforcement
+Default: yes
+Priority: 1024
+Conflicts: cracklib
+Auth-Type: Primary
+Auth-Initial:
+  required      pam_tally2.so deny={{ auth_retries }} onerr=fail unlock_time={{ auth_lockout_time }}
+Account-Type: Primary
+Account-Initial:
+  required      pam_tally2.so

hooks/charmhelpers/contrib/hardening/mysql/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from os import path
+
+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')

hooks/charmhelpers/contrib/hardening/mysql/checks/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+)
+from charmhelpers.contrib.hardening.mysql.checks import config
+
+
+def run_mysql_checks():
+    log("Starting MySQL hardening checks.", level=DEBUG)
+    checks = config.get_audits()
+    for check in checks:
+        log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
+        check.ensure_compliance()
+
+    log("MySQL hardening checks complete.", level=DEBUG)

hooks/charmhelpers/contrib/hardening/mysql/checks/config.py

@@ -0,0 +1,89 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import six
+import subprocess
+
+from charmhelpers.core.hookenv import (
+    log,
+    WARNING,
+)
+from charmhelpers.contrib.hardening.audits.file import (
+    FilePermissionAudit,
+    DirectoryPermissionAudit,
+    TemplatedFile,
+)
+from charmhelpers.contrib.hardening.mysql import TEMPLATES_DIR
+from charmhelpers.contrib.hardening import utils
+
+
+def get_audits():
+    """Get MySQL hardening config audits.
+
+    :returns:  dictionary of audits
+    """
+    if subprocess.call(['which', 'mysql'], stdout=subprocess.PIPE) != 0:
+        log("MySQL does not appear to be installed on this node - "
+            "skipping mysql hardening", level=WARNING)
+        return []
+
+    settings = utils.get_settings('mysql')
+    hardening_settings = settings['hardening']
+    my_cnf = hardening_settings['mysql-conf']
+
+    audits = [
+        FilePermissionAudit(paths=[my_cnf], user='root',
+                            group='root', mode=0o0600),
+
+        TemplatedFile(hardening_settings['hardening-conf'],
+                      MySQLConfContext(),
+                      TEMPLATES_DIR,
+                      mode=0o0750,
+                      user='mysql',
+                      group='root',
+                      service_actions=[{'service': 'mysql',
+                                        'actions': ['restart']}]),
+
+        # MySQL and Percona charms do not allow configuration of the
+        # data directory, so use the default.
+        DirectoryPermissionAudit('/var/lib/mysql',
+                                 user='mysql',
+                                 group='mysql',
+                                 recursive=False,
+                                 mode=0o755),
+
+        DirectoryPermissionAudit('/etc/mysql',
+                                 user='root',
+                                 group='root',
+                                 recursive=False,
+                                 mode=0o700),
+    ]
+
+    return audits
+
+
+class MySQLConfContext(object):
+    """Defines the set of key/value pairs to set in a mysql config file.
+
+    This context, when called, will return a dictionary containing the
+    key/value pairs of setting to specify in the
+    /etc/mysql/conf.d/hardening.cnf file.
+    """
+    def __call__(self):
+        settings = utils.get_settings('mysql')
+        # Translate for python3
+        return {'mysql_settings':
+                [(k, v) for k, v in six.iteritems(settings['security'])]}

hooks/charmhelpers/contrib/hardening/mysql/templates/hardening.cnf

@@ -0,0 +1,12 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+[mysqld]
+{% for setting, value in mysql_settings -%}
+{% if value == 'True' -%}
+{{ setting }}
+{% elif value != 'None' and value != None -%}
+{{ setting }} = {{ value }}
+{% endif -%}
+{% endfor -%}

hooks/charmhelpers/contrib/hardening/ssh/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from os import path
+
+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')

hooks/charmhelpers/contrib/hardening/ssh/checks/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+)
+from charmhelpers.contrib.hardening.ssh.checks import config
+
+
+def run_ssh_checks():
+    log("Starting SSH hardening checks.", level=DEBUG)
+    checks = config.get_audits()
+    for check in checks:
+        log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
+        check.ensure_compliance()
+
+    log("SSH hardening checks complete.", level=DEBUG)

hooks/charmhelpers/contrib/hardening/ssh/checks/config.py

@@ -0,0 +1,394 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+)
+from charmhelpers.fetch import (
+    apt_install,
+    apt_update,
+)
+from charmhelpers.core.host import lsb_release
+from charmhelpers.contrib.hardening.audits.file import (
+    TemplatedFile,
+    FileContentAudit,
+)
+from charmhelpers.contrib.hardening.ssh import TEMPLATES_DIR
+from charmhelpers.contrib.hardening import utils
+
+
+def get_audits():
+    """Get SSH hardening config audits.
+
+    :returns:  dictionary of audits
+    """
+    audits = [SSHConfig(), SSHDConfig(), SSHConfigFileContentAudit(),
+              SSHDConfigFileContentAudit()]
+    return audits
+
+
+class SSHConfigContext(object):
+
+    type = 'client'
+
+    def get_macs(self, allow_weak_mac):
+        if allow_weak_mac:
+            weak_macs = 'weak'
+        else:
+            weak_macs = 'default'
+
+        default = 'hmac-sha2-512,hmac-sha2-256,hmac-ripemd160'
+        macs = {'default': default,
+                'weak': default + ',hmac-sha1'}
+
+        default = ('hmac-sha2-512-etm@openssh.com,'
+                   'hmac-sha2-256-etm@openssh.com,'
+                   'hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,'
+                   'hmac-sha2-512,hmac-sha2-256,hmac-ripemd160')
+        macs_66 = {'default': default,
+                   'weak': default + ',hmac-sha1'}
+
+        # Use newer ciphers on Ubuntu Trusty and above
+        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+            log("Detected Ubuntu 14.04 or newer, using new macs", level=DEBUG)
+            macs = macs_66
+
+        return macs[weak_macs]
+
+    def get_kexs(self, allow_weak_kex):
+        if allow_weak_kex:
+            weak_kex = 'weak'
+        else:
+            weak_kex = 'default'
+
+        default = 'diffie-hellman-group-exchange-sha256'
+        weak = (default + ',diffie-hellman-group14-sha1,'
+                'diffie-hellman-group-exchange-sha1,'
+                'diffie-hellman-group1-sha1')
+        kex = {'default': default,
+               'weak': weak}
+
+        default = ('curve25519-sha256@libssh.org,'
+                   'diffie-hellman-group-exchange-sha256')
+        weak = (default + ',diffie-hellman-group14-sha1,'
+                'diffie-hellman-group-exchange-sha1,'
+                'diffie-hellman-group1-sha1')
+        kex_66 = {'default': default,
+                  'weak': weak}
+
+        # Use newer kex on Ubuntu Trusty and above
+        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+            log('Detected Ubuntu 14.04 or newer, using new key exchange '
+                'algorithms', level=DEBUG)
+            kex = kex_66
+
+        return kex[weak_kex]
+
+    def get_ciphers(self, cbc_required):
+        if cbc_required:
+            weak_ciphers = 'weak'
+        else:
+            weak_ciphers = 'default'
+
+        default = 'aes256-ctr,aes192-ctr,aes128-ctr'
+        cipher = {'default': default,
+                  'weak': default + 'aes256-cbc,aes192-cbc,aes128-cbc'}
+
+        default = ('chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,'
+                   'aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr')
+        ciphers_66 = {'default': default,
+                      'weak': default + ',aes256-cbc,aes192-cbc,aes128-cbc'}
+
+        # Use newer ciphers on ubuntu Trusty and above
+        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+            log('Detected Ubuntu 14.04 or newer, using new ciphers',
+                level=DEBUG)
+            cipher = ciphers_66
+
+        return cipher[weak_ciphers]
+
+    def __call__(self):
+        settings = utils.get_settings('ssh')
+        if settings['common']['network_ipv6_enable']:
+            addr_family = 'any'
+        else:
+            addr_family = 'inet'
+
+        ctxt = {
+            'addr_family': addr_family,
+            'remote_hosts': settings['common']['remote_hosts'],
+            'password_auth_allowed':
+            settings['client']['password_authentication'],
+            'ports': settings['common']['ports'],
+            'ciphers': self.get_ciphers(settings['client']['cbc_required']),
+            'macs': self.get_macs(settings['client']['weak_hmac']),
+            'kexs': self.get_kexs(settings['client']['weak_kex']),
+            'roaming': settings['client']['roaming'],
+        }
+        return ctxt
+
+
+class SSHConfig(TemplatedFile):
+    def __init__(self):
+        path = '/etc/ssh/ssh_config'
+        super(SSHConfig, self).__init__(path=path,
+                                        template_dir=TEMPLATES_DIR,
+                                        context=SSHConfigContext(),
+                                        user='root',
+                                        group='root',
+                                        mode=0o0644)
+
+    def pre_write(self):
+        settings = utils.get_settings('ssh')
+        apt_update(fatal=True)
+        apt_install(settings['client']['package'])
+        if not os.path.exists('/etc/ssh'):
+            os.makedir('/etc/ssh')
+            # NOTE: don't recurse
+            utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
+                                     maxdepth=0)
+
+    def post_write(self):
+        # NOTE: don't recurse
+        utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
+                                 maxdepth=0)
+
+
+class SSHDConfigContext(SSHConfigContext):
+
+    type = 'server'
+
+    def __call__(self):
+        settings = utils.get_settings('ssh')
+        if settings['common']['network_ipv6_enable']:
+            addr_family = 'any'
+        else:
+            addr_family = 'inet'
+
+        ctxt = {
+            'ssh_ip': settings['server']['listen_to'],
+            'password_auth_allowed':
+            settings['server']['password_authentication'],
+            'ports': settings['common']['ports'],
+            'addr_family': addr_family,
+            'ciphers': self.get_ciphers(settings['server']['cbc_required']),
+            'macs': self.get_macs(settings['server']['weak_hmac']),
+            'kexs': self.get_kexs(settings['server']['weak_kex']),
+            'host_key_files': settings['server']['host_key_files'],
+            'allow_root_with_key': settings['server']['allow_root_with_key'],
+            'password_authentication':
+            settings['server']['password_authentication'],
+            'use_priv_sep': settings['server']['use_privilege_separation'],
+            'use_pam': settings['server']['use_pam'],
+            'allow_x11_forwarding': settings['server']['allow_x11_forwarding'],
+            'print_motd': settings['server']['print_motd'],
+            'print_last_log': settings['server']['print_last_log'],
+            'client_alive_interval':
+            settings['server']['alive_interval'],
+            'client_alive_count': settings['server']['alive_count'],
+            'allow_tcp_forwarding': settings['server']['allow_tcp_forwarding'],
+            'allow_agent_forwarding':
+            settings['server']['allow_agent_forwarding'],
+            'deny_users': settings['server']['deny_users'],
+            'allow_users': settings['server']['allow_users'],
+            'deny_groups': settings['server']['deny_groups'],
+            'allow_groups': settings['server']['allow_groups'],
+            'use_dns': settings['server']['use_dns'],
+            'sftp_enable': settings['server']['sftp_enable'],
+            'sftp_group': settings['server']['sftp_group'],
+            'sftp_chroot': settings['server']['sftp_chroot'],
+            'max_auth_tries': settings['server']['max_auth_tries'],
+            'max_sessions': settings['server']['max_sessions'],
+        }
+        return ctxt
+
+
+class SSHDConfig(TemplatedFile):
+    def __init__(self):
+        path = '/etc/ssh/sshd_config'
+        super(SSHDConfig, self).__init__(path=path,
+                                         template_dir=TEMPLATES_DIR,
+                                         context=SSHDConfigContext(),
+                                         user='root',
+                                         group='root',
+                                         mode=0o0600,
+                                         service_actions=[{'service': 'ssh',
+                                                           'actions':
+                                                           ['restart']}])
+
+    def pre_write(self):
+        settings = utils.get_settings('ssh')
+        apt_update(fatal=True)
+        apt_install(settings['server']['package'])
+        if not os.path.exists('/etc/ssh'):
+            os.makedir('/etc/ssh')
+            # NOTE: don't recurse
+            utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
+                                     maxdepth=0)
+
+    def post_write(self):
+        # NOTE: don't recurse
+        utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
+                                 maxdepth=0)
+
+
+class SSHConfigFileContentAudit(FileContentAudit):
+    def __init__(self):
+        self.path = '/etc/ssh/ssh_config'
+        super(SSHConfigFileContentAudit, self).__init__(self.path, {})
+
+    def is_compliant(self, *args, **kwargs):
+        self.pass_cases = []
+        self.fail_cases = []
+        settings = utils.get_settings('ssh')
+
+        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+            if not settings['server']['weak_hmac']:
+                self.pass_cases.append(r'^MACs.+,hmac-ripemd160$')
+            else:
+                self.pass_cases.append(r'^MACs.+,hmac-sha1$')
+
+            if settings['server']['weak_kex']:
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?')  # noqa
+            else:
+                self.pass_cases.append(r'^KexAlgorithms.+,diffie-hellman-group-exchange-sha256$')  # noqa
+                self.fail_cases.append(r'^KexAlgorithms.*diffie-hellman-group14-sha1[,\s]?')  # noqa
+
+            if settings['server']['cbc_required']:
+                self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
+            else:
+                self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
+                self.pass_cases.append(r'^Ciphers\schacha20-poly1305@openssh.com,.+')  # noqa
+                self.pass_cases.append(r'^Ciphers\s.*aes128-ctr$')
+                self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
+                self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
+        else:
+            if not settings['client']['weak_hmac']:
+                self.fail_cases.append(r'^MACs.+,hmac-sha1$')
+            else:
+                self.pass_cases.append(r'^MACs.+,hmac-sha1$')
+
+            if settings['client']['weak_kex']:
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?')  # noqa
+            else:
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256$')  # noqa
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?')  # noqa
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?')  # noqa
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?')  # noqa
+
+            if settings['client']['cbc_required']:
+                self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
+            else:
+                self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
+                self.pass_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
+                self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
+                self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
+
+        if settings['client']['roaming']:
+            self.pass_cases.append(r'^UseRoaming yes$')
+        else:
+            self.fail_cases.append(r'^UseRoaming yes$')
+
+        return super(SSHConfigFileContentAudit, self).is_compliant(*args,
+                                                                   **kwargs)
+
+
+class SSHDConfigFileContentAudit(FileContentAudit):
+    def __init__(self):
+        self.path = '/etc/ssh/sshd_config'
+        super(SSHDConfigFileContentAudit, self).__init__(self.path, {})
+
+    def is_compliant(self, *args, **kwargs):
+        self.pass_cases = []
+        self.fail_cases = []
+        settings = utils.get_settings('ssh')
+
+        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+            if not settings['server']['weak_hmac']:
+                self.pass_cases.append(r'^MACs.+,hmac-ripemd160$')
+            else:
+                self.pass_cases.append(r'^MACs.+,hmac-sha1$')
+
+            if settings['server']['weak_kex']:
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?')  # noqa
+            else:
+                self.pass_cases.append(r'^KexAlgorithms.+,diffie-hellman-group-exchange-sha256$')  # noqa
+                self.fail_cases.append(r'^KexAlgorithms.*diffie-hellman-group14-sha1[,\s]?')  # noqa
+
+            if settings['server']['cbc_required']:
+                self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
+            else:
+                self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
+                self.pass_cases.append(r'^Ciphers\schacha20-poly1305@openssh.com,.+')  # noqa
+                self.pass_cases.append(r'^Ciphers\s.*aes128-ctr$')
+                self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
+                self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
+        else:
+            if not settings['server']['weak_hmac']:
+                self.pass_cases.append(r'^MACs.+,hmac-ripemd160$')
+            else:
+                self.pass_cases.append(r'^MACs.+,hmac-sha1$')
+
+            if settings['server']['weak_kex']:
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?')  # noqa
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?')  # noqa
+            else:
+                self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256$')  # noqa
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?')  # noqa
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?')  # noqa
+                self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?')  # noqa
+
+            if settings['server']['cbc_required']:
+                self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
+                self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
+            else:
+                self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
+                self.pass_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
+                self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
+                self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
+
+        if settings['server']['sftp_enable']:
+            self.pass_cases.append(r'^Subsystem\ssftp')
+        else:
+            self.fail_cases.append(r'^Subsystem\ssftp')
+
+        return super(SSHDConfigFileContentAudit, self).is_compliant(*args,
+                                                                    **kwargs)

hooks/charmhelpers/contrib/hardening/ssh/templates/ssh_config

@@ -0,0 +1,70 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#       be overwritten.
+###############################################################################
+# This is the ssh client system-wide configuration file.  See
+# ssh_config(5) for more information.  This file provides defaults for
+# users, and the values can be changed in per-user configuration files
+# or on the command line.
+
+# Configuration data is parsed as follows:
+#  1. command line options
+#  2. user-specific file
+#  3. system-wide file
+# Any configuration value is only changed the first time it is set.
+# Thus, host-specific definitions should be at the beginning of the
+# configuration file, and defaults at the end.
+
+# Site-wide defaults for some commonly used options.  For a comprehensive
+# list of available options, their meanings and defaults, please see the
+# ssh_config(5) man page.
+
+# Restrict the following configuration to be limited to this Host.
+{% if remote_hosts -%}
+Host {{ ' '.join(remote_hosts) }}
+{% endif %}
+ForwardAgent no
+ForwardX11 no
+ForwardX11Trusted yes
+RhostsRSAAuthentication no
+RSAAuthentication yes
+PasswordAuthentication {{ password_auth_allowed }}
+HostbasedAuthentication no
+GSSAPIAuthentication no
+GSSAPIDelegateCredentials no
+GSSAPIKeyExchange no
+GSSAPITrustDNS no
+BatchMode no
+CheckHostIP yes
+AddressFamily {{ addr_family }}
+ConnectTimeout 0
+StrictHostKeyChecking ask
+IdentityFile ~/.ssh/identity
+IdentityFile ~/.ssh/id_rsa
+IdentityFile ~/.ssh/id_dsa
+# The port at the destination should be defined
+{% for port in ports -%}
+Port {{ port }}
+{% endfor %}
+Protocol 2
+Cipher 3des
+{% if ciphers -%}
+Ciphers {{ ciphers }}
+{%- endif %}
+{% if macs -%}
+MACs {{ macs }}
+{%- endif %}
+{% if kexs -%}
+KexAlgorithms {{ kexs }}
+{%- endif %}
+EscapeChar ~
+Tunnel no
+TunnelDevice any:any
+PermitLocalCommand no
+VisualHostKey no
+RekeyLimit 1G 1h
+SendEnv LANG LC_*
+HashKnownHosts yes
+{% if roaming -%}
+UseRoaming {{ roaming }}
+{% endif %}

hooks/charmhelpers/contrib/hardening/ssh/templates/sshd_config

@@ -0,0 +1,159 @@
+###############################################################################
+# WARNING: This configuration file is maintained by Juju. Local changes may
+#          be overwritten.
+###############################################################################
+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+# What ports, IPs and protocols we listen for
+{% for port in ports -%}
+Port {{ port }}
+{% endfor -%}
+AddressFamily {{ addr_family }}
+# Use these options to restrict which interfaces/protocols sshd will bind to
+{% if ssh_ip -%}
+{% for ip in ssh_ip -%}
+ListenAddress {{ ip }}
+{% endfor %}
+{%- else -%}
+ListenAddress ::
+ListenAddress 0.0.0.0
+{% endif -%}
+Protocol 2
+{% if ciphers -%}
+Ciphers {{ ciphers }}
+{% endif -%}
+{% if macs -%}
+MACs {{ macs }}
+{% endif -%}
+{% if kexs -%}
+KexAlgorithms {{ kexs }}
+{% endif -%}
+# HostKeys for protocol version 2
+{% for keyfile in host_key_files -%}
+HostKey {{ keyfile }}
+{% endfor -%}
+
+# Privilege Separation is turned on for security
+{% if use_priv_sep -%} 
+UsePrivilegeSeparation {{ use_priv_sep }}
+{% endif -%}
+
+# Lifetime and size of ephemeral version 1 server key
+KeyRegenerationInterval 3600
+ServerKeyBits 1024
+
+# Logging
+SyslogFacility AUTH
+LogLevel VERBOSE
+
+# Authentication:
+LoginGraceTime 30s
+{% if allow_root_with_key -%}
+PermitRootLogin without-password
+{% else -%}
+PermitRootLogin no
+{% endif %}
+PermitTunnel no
+PermitUserEnvironment no
+StrictModes yes
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+AuthorizedKeysFile %h/.ssh/authorized_keys
+
+# Don't read the user's ~/.rhosts and ~/.shosts files
+IgnoreRhosts yes
+# For this to work you will also need host keys in /etc/ssh_known_hosts
+RhostsRSAAuthentication no
+# similar for protocol version 2
+HostbasedAuthentication no
+# Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication
+IgnoreUserKnownHosts yes
+
+# To enable empty passwords, change to yes (NOT RECOMMENDED)
+PermitEmptyPasswords no
+
+# Change to yes to enable challenge-response passwords (beware issues with
+# some PAM modules and threads)
+ChallengeResponseAuthentication no
+
+# Change to no to disable tunnelled clear text passwords
+PasswordAuthentication {{ password_authentication }}
+
+# Kerberos options
+KerberosAuthentication no
+KerberosGetAFSToken no
+KerberosOrLocalPasswd no
+KerberosTicketCleanup yes
+
+# GSSAPI options
+GSSAPIAuthentication no
+GSSAPICleanupCredentials yes
+
+X11Forwarding {{ allow_x11_forwarding }}
+X11DisplayOffset 10
+X11UseLocalhost yes
+GatewayPorts no
+PrintMotd {{ print_motd }}
+PrintLastLog {{ print_last_log }}
+TCPKeepAlive no
+UseLogin no
+
+ClientAliveInterval {{ client_alive_interval }}
+ClientAliveCountMax {{ client_alive_count }}
+AllowTcpForwarding {{ allow_tcp_forwarding }}
+AllowAgentForwarding {{ allow_agent_forwarding }}
+
+MaxStartups 10:30:100
+#Banner /etc/issue.net
+
+# Allow client to pass locale environment variables
+AcceptEnv LANG LC_*
+
+# Set this to 'yes' to enable PAM authentication, account processing,
+# and session processing. If this is enabled, PAM authentication will
+# be allowed through the ChallengeResponseAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via ChallengeResponseAuthentication may bypass
+# the setting of "PermitRootLogin without-password".
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and ChallengeResponseAuthentication to 'no'.
+UsePAM {{ use_pam }}
+
+{% if deny_users -%}
+DenyUsers {{ deny_users }}
+{% endif -%}
+{% if allow_users -%}
+AllowUsers {{ allow_users }}
+{% endif -%}
+{% if deny_groups -%}
+DenyGroups {{ deny_groups }}
+{% endif -%}
+{% if allow_groups -%}
+AllowGroups allow_groups
+{% endif -%}
+UseDNS {{ use_dns }}
+MaxAuthTries {{ max_auth_tries }}
+MaxSessions {{ max_sessions }}
+
+{% if sftp_enable -%}
+# Configuration, in case SFTP is used
+## override default of no subsystems
+## Subsystem sftp /opt/app/openssh5/libexec/sftp-server
+Subsystem sftp internal-sftp -l VERBOSE
+
+## These lines must appear at the *end* of sshd_config
+Match Group {{ sftp_group }}
+ForceCommand internal-sftp -l VERBOSE
+ChrootDirectory {{ sftp_chroot }}
+{% else -%}
+# Configuration, in case SFTP is used
+## override default of no subsystems
+## Subsystem sftp /opt/app/openssh5/libexec/sftp-server
+## These lines must appear at the *end* of sshd_config
+Match Group sftponly
+ForceCommand internal-sftp -l VERBOSE
+ChrootDirectory /sftpchroot/home/%u
+{% endif %}

hooks/charmhelpers/contrib/hardening/templating.py

@@ -0,0 +1,71 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+    WARNING,
+)
+
+try:
+    from jinja2 import FileSystemLoader, Environment
+except ImportError:
+    from charmhelpers.fetch import apt_install
+    from charmhelpers.fetch import apt_update
+    apt_update(fatal=True)
+    apt_install('python-jinja2', fatal=True)
+    from jinja2 import FileSystemLoader, Environment
+
+
+# NOTE: function separated from main rendering code to facilitate easier
+#       mocking in unit tests.
+def write(path, data):
+    with open(path, 'wb') as out:
+        out.write(data)
+
+
+def get_template_path(template_dir, path):
+    """Returns the template file which would be used to render the path.
+
+    The path to the template file is returned.
+    :param template_dir: the directory the templates are located in
+    :param path: the file path to be written to.
+    :returns: path to the template file
+    """
+    return os.path.join(template_dir, os.path.basename(path))
+
+
+def render_and_write(template_dir, path, context):
+    """Renders the specified template into the file.
+
+    :param template_dir: the directory to load the template from
+    :param path: the path to write the templated contents to
+    :param context: the parameters to pass to the rendering engine
+    """
+    env = Environment(loader=FileSystemLoader(template_dir))
+    template_file = os.path.basename(path)
+    template = env.get_template(template_file)
+    log('Rendering from template: %s' % template.name, level=DEBUG)
+    rendered_content = template.render(context)
+    if not rendered_content:
+        log("Render returned None - skipping '%s'" % path,
+            level=WARNING)
+        return
+
+    write(path, rendered_content.encode('utf-8').strip())
+    log('Wrote template %s' % path, level=DEBUG)

hooks/charmhelpers/contrib/hardening/utils.py

@@ -0,0 +1,157 @@
+# Copyright 2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import glob
+import grp
+import os
+import pwd
+import six
+import yaml
+
+from charmhelpers.core.hookenv import (
+    log,
+    DEBUG,
+    INFO,
+    WARNING,
+    ERROR,
+)
+
+
+# Global settings cache. Since each hook fire entails a fresh module import it
+# is safe to hold this in memory and not risk missing config changes (since
+# they will result in a new hook fire and thus re-import).
+__SETTINGS__ = {}
+
+
+def _get_defaults(modules):
+    """Load the default config for the provided modules.
+
+    :param modules: stack modules config defaults to lookup.
+    :returns: modules default config dictionary.
+    """
+    default = os.path.join(os.path.dirname(__file__),
+                           'defaults/%s.yaml' % (modules))
+    return yaml.safe_load(open(default))
+
+
+def _get_schema(modules):
+    """Load the config schema for the provided modules.
+
+    NOTE: this schema is intended to have 1-1 relationship with they keys in
+    the default config and is used a means to verify valid overrides provided
+    by the user.
+
+    :param modules: stack modules config schema to lookup.
+    :returns: modules default schema dictionary.
+    """
+    schema = os.path.join(os.path.dirname(__file__),
+                          'defaults/%s.yaml.schema' % (modules))
+    return yaml.safe_load(open(schema))
+
+
+def _get_user_provided_overrides(modules):
+    """Load user-provided config overrides.
+
+    :param modules: stack modules to lookup in user overrides yaml file.
+    :returns: overrides dictionary.
+    """
+    overrides = os.path.join(os.environ['JUJU_CHARM_DIR'],
+                             'hardening.yaml')
+    if os.path.exists(overrides):
+        log("Found user-provided config overrides file '%s'" %
+            (overrides), level=DEBUG)
+        settings = yaml.safe_load(open(overrides))
+        if settings and settings.get(modules):
+            log("Applying '%s' overrides" % (modules), level=DEBUG)
+            return settings.get(modules)
+
+        log("No overrides found for '%s'" % (modules), level=DEBUG)
+    else:
+        log("No hardening config overrides file '%s' found in charm "
+            "root dir" % (overrides), level=DEBUG)
+
+    return {}
+
+
+def _apply_overrides(settings, overrides, schema):
+    """Get overrides config overlayed onto modules defaults.
+
+    :param modules: require stack modules config.
+    :returns: dictionary of modules config with user overrides applied.
+    """
+    if overrides:
+        for k, v in six.iteritems(overrides):
+            if k in schema:
+                if schema[k] is None:
+                    settings[k] = v
+                elif type(schema[k]) is dict:
+                    settings[k] = _apply_overrides(settings[k], overrides[k],
+                                                   schema[k])
+                else:
+                    raise Exception("Unexpected type found in schema '%s'" %
+                                    type(schema[k]), level=ERROR)
+            else:
+                log("Unknown override key '%s' - ignoring" % (k), level=INFO)
+
+    return settings
+
+
+def get_settings(modules):
+    global __SETTINGS__
+    if modules in __SETTINGS__:
+        return __SETTINGS__[modules]
+
+    schema = _get_schema(modules)
+    settings = _get_defaults(modules)
+    overrides = _get_user_provided_overrides(modules)
+    __SETTINGS__[modules] = _apply_overrides(settings, overrides, schema)
+    return __SETTINGS__[modules]
+
+
+def ensure_permissions(path, user, group, permissions, maxdepth=-1):
+    """Ensure permissions for path.
+
+    If path is a file, apply to file and return. If path is a directory,
+    apply recursively (if required) to directory contents and return.
+
+    :param user: user name
+    :param group: group name
+    :param permissions: octal permissions
+    :param maxdepth: maximum recursion depth. A negative maxdepth allows
+                     infinite recursion and maxdepth=0 means no recursion.
+    :returns: None
+    """
+    if not os.path.exists(path):
+        log("File '%s' does not exist - cannot set permissions" % (path),
+            level=WARNING)
+        return
+
+    _user = pwd.getpwnam(user)
+    os.chown(path, _user.pw_uid, grp.getgrnam(group).gr_gid)
+    os.chmod(path, permissions)
+
+    if maxdepth == 0:
+        log("Max recursion depth reached - skipping further recursion",
+            level=DEBUG)
+        return
+    elif maxdepth > 0:
+        maxdepth -= 1
+
+    if os.path.isdir(path):
+        contents = glob.glob("%s/*" % (path))
+        for c in contents:
+            ensure_permissions(c, user=user, group=group,
+                               permissions=permissions, maxdepth=maxdepth)

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -27,7 +27,11 @@ import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
-import novaclient.v1_1.client as nova_client
+from keystoneclient.auth.identity import v3 as keystone_id_v3
+from keystoneclient import session as keystone_session
+from keystoneclient.v3 import client as keystone_client_v3
+
+import novaclient.client as nova_client
 import pika
 import swiftclient
 
@@ -38,6 +42,8 @@ from charmhelpers.contrib.amulet.utils import (
 DEBUG = logging.DEBUG
 ERROR = logging.ERROR
 
+NOVA_CLIENT_VERSION = "2"
+
 
 class OpenStackAmuletUtils(AmuletUtils):
     """OpenStack amulet utilities.
@@ -139,7 +145,7 @@ class OpenStackAmuletUtils(AmuletUtils):
                 return "role {} does not exist".format(e['name'])
         return ret
 
-    def validate_user_data(self, expected, actual):
+    def validate_user_data(self, expected, actual, api_version=None):
         """Validate user data.
 
            Validate a list of actual user data vs a list of expected user
@@ -150,10 +156,15 @@ class OpenStackAmuletUtils(AmuletUtils):
         for e in expected:
             found = False
             for act in actual:
-                a = {'enabled': act.enabled, 'name': act.name,
-                     'email': act.email, 'tenantId': act.tenantId,
-                     'id': act.id}
-                if e['name'] == a['name']:
+                if e['name'] == act.name:
+                    a = {'enabled': act.enabled, 'name': act.name,
+                         'email': act.email, 'id': act.id}
+                    if api_version == 3:
+                        a['default_project_id'] = getattr(act,
+                                                          'default_project_id',
+                                                          'none')
+                    else:
+                        a['tenantId'] = act.tenantId
                     found = True
                     ret = self._validate_dict_data(e, a)
                     if ret:
@@ -188,15 +199,30 @@ class OpenStackAmuletUtils(AmuletUtils):
         return cinder_client.Client(username, password, tenant, ept)
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
-                                    tenant):
+                                    tenant=None, api_version=None,
+                                    keystone_ip=None):
         """Authenticates admin user with the keystone admin endpoint."""
         self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
-        service_ip = unit.relation('shared-db',
-                                   'mysql:shared-db')['private-address']
-        ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
-        return keystone_client.Client(username=user, password=password,
-                                      tenant_name=tenant, auth_url=ep)
+        if not keystone_ip:
+            keystone_ip = unit.relation('shared-db',
+                                        'mysql:shared-db')['private-address']
+        base_ep = "http://{}:35357".format(keystone_ip.strip().decode('utf-8'))
+        if not api_version or api_version == 2:
+            ep = base_ep + "/v2.0"
+            return keystone_client.Client(username=user, password=password,
+                                          tenant_name=tenant, auth_url=ep)
+        else:
+            ep = base_ep + "/v3"
+            auth = keystone_id_v3.Password(
+                user_domain_name='admin_domain',
+                username=user,
+                password=password,
+                domain_name='admin_domain',
+                auth_url=ep,
+            )
+            sess = keystone_session.Session(auth=auth)
+            return keystone_client_v3.Client(session=sess)
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
@@ -225,7 +251,8 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return nova_client.Client(username=user, api_key=password,
+        return nova_client.Client(NOVA_CLIENT_VERSION,
+                                  username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
     def authenticate_swift_user(self, keystone, user, password, tenant):

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -1,20 +1,12 @@
 {% if auth_host -%}
-{% if api_version == '3' -%}
 [keystone_authtoken]
-auth_url = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
+auth_plugin = password
+project_domain_id = default
+user_domain_id = default
 project_name = {{ admin_tenant_name }}
 username = {{ admin_user }}
 password = {{ admin_password }}
-project_domain_name = default
-user_domain_name = default
-auth_plugin = password
-{% else -%}
-[keystone_authtoken]
-identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
-auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
-admin_tenant_name = {{ admin_tenant_name }}
-admin_user = {{ admin_user }}
-admin_password = {{ admin_password }}
 signing_dir = {{ signing_dir }}
 {% endif -%}
-{% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy

@@ -0,0 +1,10 @@
+{% if auth_host -%}
+[keystone_authtoken]
+# Juno specific config (Bug #1557223)
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
+identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
+admin_tenant_name = {{ admin_tenant_name }}
+admin_user = {{ admin_user }}
+admin_password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -24,6 +24,7 @@ import os
 import sys
 import re
 import itertools
+import functools
 
 import six
 import tempfile
@@ -69,7 +70,15 @@ from charmhelpers.contrib.python.packages import (
     pip_install,
 )
 
-from charmhelpers.core.host import lsb_release, mounts, umount, service_running
+from charmhelpers.core.host import (
+    lsb_release,
+    mounts,
+    umount,
+    service_running,
+    service_pause,
+    service_resume,
+    restart_on_change_helper,
+)
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
@@ -128,7 +137,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('liberty',
         ['2.3.0', '2.4.0', '2.5.0']),
     ('mitaka',
-        ['2.5.0']),
+        ['2.5.0', '2.6.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -763,7 +772,8 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
         os.mkdir(parent_dir)
 
     juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-    repo_dir = install_remote(repo, dest=parent_dir, branch=branch, depth=depth)
+    repo_dir = install_remote(
+        repo, dest=parent_dir, branch=branch, depth=depth)
 
     venv = os.path.join(parent_dir, 'venv')
 
@@ -862,66 +872,155 @@ def os_workload_status(configs, required_interfaces, charm_func=None):
     return wrap
 
 
-def set_os_workload_status(configs, required_interfaces, charm_func=None, services=None, ports=None):
-    """
-    Set workload status based on complete contexts.
-    status-set missing or incomplete contexts
-    and juju-log details of missing required data.
-    charm_func is a charm specific function to run checking
-    for charm specific requirements such as a VIP setting.
+def set_os_workload_status(configs, required_interfaces, charm_func=None,
+                           services=None, ports=None):
+    """Set the state of the workload status for the charm.
 
-    This function also checks for whether the services defined are ACTUALLY
-    running and that the ports they advertise are open and being listened to.
+    This calls _determine_os_workload_status() to get the new state, message
+    and sets the status using status_set()
 
-    @param services - OPTIONAL: a [{'service': <string>, 'ports': [<int>]]
-                      The ports are optional.
-                      If services is a [<string>] then ports are ignored.
-    @param ports - OPTIONAL: an [<int>] representing ports that shoudl be
-                   open.
-    @returns None
+    @param configs: a templating.OSConfigRenderer() object
+    @param required_interfaces: {generic: [specific, specific2, ...]}
+    @param charm_func: a callable function that returns state, message. The
+                       signature is charm_func(configs) -> (state, message)
+    @param services: list of strings OR dictionary specifying services/ports
+    @param ports: OPTIONAL list of port numbers.
+    @returns state, message: the new workload status, user message
     """
-    incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
-    state = 'active'
-    missing_relations = []
-    incomplete_relations = []
+    state, message = _determine_os_workload_status(
+        configs, required_interfaces, charm_func, services, ports)
+    status_set(state, message)
+
+
+def _determine_os_workload_status(
+        configs, required_interfaces, charm_func=None,
+        services=None, ports=None):
+    """Determine the state of the workload status for the charm.
+
+    This function returns the new workload status for the charm based
+    on the state of the interfaces, the paused state and whether the
+    services are actually running and any specified ports are open.
+
+    This checks:
+
+     1. if the unit should be paused, that it is actually paused.  If so the
+        state is 'maintenance' + message, else 'broken'.
+     2. that the interfaces/relations are complete.  If they are not then
+        it sets the state to either 'broken' or 'waiting' and an appropriate
+        message.
+     3. If all the relation data is set, then it checks that the actual
+        services really are running.  If not it sets the state to 'broken'.
+
+    If everything is okay then the state returns 'active'.
+
+    @param configs: a templating.OSConfigRenderer() object
+    @param required_interfaces: {generic: [specific, specific2, ...]}
+    @param charm_func: a callable function that returns state, message. The
+                       signature is charm_func(configs) -> (state, message)
+    @param services: list of strings OR dictionary specifying services/ports
+    @param ports: OPTIONAL list of port numbers.
+    @returns state, message: the new workload status, user message
+    """
+    state, message = _ows_check_if_paused(services, ports)
+
+    if state is None:
+        state, message = _ows_check_generic_interfaces(
+            configs, required_interfaces)
+
+    if state != 'maintenance' and charm_func:
+        # _ows_check_charm_func() may modify the state, message
+        state, message = _ows_check_charm_func(
+            state, message, lambda: charm_func(configs))
+
+    if state is None:
+        state, message = _ows_check_services_running(services, ports)
+
+    if state is None:
+        state = 'active'
+        message = "Unit is ready"
+        juju_log(message, 'INFO')
+
+    return state, message
+
+
+def _ows_check_if_paused(services=None, ports=None):
+    """Check if the unit is supposed to be paused, and if so check that the
+    services/ports (if passed) are actually stopped/not being listened to.
+
+    if the unit isn't supposed to be paused, just return None, None
+
+    @param services: OPTIONAL services spec or list of service names.
+    @param ports: OPTIONAL list of port numbers.
+    @returns state, message or None, None
+    """
+    if is_unit_paused_set():
+        state, message = check_actually_paused(services=services,
+                                               ports=ports)
+        if state is None:
+            # we're paused okay, so set maintenance and return
+            state = "maintenance"
+            message = "Paused. Use 'resume' action to resume normal service."
+        return state, message
+    return None, None
+
+
+def _ows_check_generic_interfaces(configs, required_interfaces):
+    """Check the complete contexts to determine the workload status.
+
+     - Checks for missing or incomplete contexts
+     - juju log details of missing required data.
+     - determines the correct workload status
+     - creates an appropriate message for status_set(...)
+
+    if there are no problems then the function returns None, None
+
+    @param configs: a templating.OSConfigRenderer() object
+    @params required_interfaces: {generic_interface: [specific_interface], }
+    @returns state, message or None, None
+    """
+    incomplete_rel_data = incomplete_relation_data(configs,
+                                                   required_interfaces)
+    state = None
     message = None
-    charm_state = None
-    charm_message = None
+    missing_relations = set()
+    incomplete_relations = set()
 
-    for generic_interface in incomplete_rel_data.keys():
+    for generic_interface, relations_states in incomplete_rel_data.items():
         related_interface = None
         missing_data = {}
         # Related or not?
-        for interface in incomplete_rel_data[generic_interface]:
-            if incomplete_rel_data[generic_interface][interface].get('related'):
+        for interface, relation_state in relations_states.items():
+            if relation_state.get('related'):
                 related_interface = interface
-                missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
-        # No relation ID for the generic_interface
+                missing_data = relation_state.get('missing_data')
+                break
+        # No relation ID for the generic_interface?
         if not related_interface:
             juju_log("{} relation is missing and must be related for "
                      "functionality. ".format(generic_interface), 'WARN')
             state = 'blocked'
-            if generic_interface not in missing_relations:
-                missing_relations.append(generic_interface)
+            missing_relations.add(generic_interface)
         else:
-            # Relation ID exists but no related unit
+            # Relation ID eists but no related unit
             if not missing_data:
-                # Edge case relation ID exists but departing
-                if ('departed' in hook_name() or 'broken' in hook_name()) \
-                        and related_interface in hook_name():
+                # Edge case - relation ID exists but departings
+                _hook_name = hook_name()
+                if (('departed' in _hook_name or 'broken' in _hook_name) and
+                        related_interface in _hook_name):
                     state = 'blocked'
-                    if generic_interface not in missing_relations:
-                        missing_relations.append(generic_interface)
+                    missing_relations.add(generic_interface)
                     juju_log("{} relation's interface, {}, "
                              "relationship is departed or broken "
                              "and is required for functionality."
-                             "".format(generic_interface, related_interface), "WARN")
+                             "".format(generic_interface, related_interface),
+                             "WARN")
                 # Normal case relation ID exists but no related unit
                 # (joining)
                 else:
-                    juju_log("{} relations's interface, {}, is related but has "
-                             "no units in the relation."
-                             "".format(generic_interface, related_interface), "INFO")
+                    juju_log("{} relations's interface, {}, is related but has"
+                             " no units in the relation."
+                             "".format(generic_interface, related_interface),
+                             "INFO")
             # Related unit exists and data missing on the relation
             else:
                 juju_log("{} relation's interface, {}, is related awaiting "
@@ -930,9 +1029,8 @@ def set_os_workload_status(configs, required_interfaces, charm_func=None, servic
                                    ", ".join(missing_data)), "INFO")
             if state != 'blocked':
                 state = 'waiting'
-            if generic_interface not in incomplete_relations \
-                    and generic_interface not in missing_relations:
-                incomplete_relations.append(generic_interface)
+            if generic_interface not in missing_relations:
+                incomplete_relations.add(generic_interface)
 
     if missing_relations:
         message = "Missing relations: {}".format(", ".join(missing_relations))
@@ -945,9 +1043,22 @@ def set_os_workload_status(configs, required_interfaces, charm_func=None, servic
                   "".format(", ".join(incomplete_relations))
         state = 'waiting'
 
-    # Run charm specific checks
-    if charm_func:
-        charm_state, charm_message = charm_func(configs)
+    return state, message
+
+
+def _ows_check_charm_func(state, message, charm_func_with_configs):
+    """Run a custom check function for the charm to see if it wants to
+    change the state.  This is only run if not in 'maintenance' and
+    tests to see if the new state is more important that the previous
+    one determined by the interfaces/relations check.
+
+    @param state: the previously determined state so far.
+    @param message: the user orientated message so far.
+    @param charm_func: a callable function that returns state, message
+    @returns state, message strings.
+    """
+    if charm_func_with_configs:
+        charm_state, charm_message = charm_func_with_configs()
         if charm_state != 'active' and charm_state != 'unknown':
             state = workload_state_compare(state, charm_state)
             if message:
@@ -956,72 +1067,151 @@ def set_os_workload_status(configs, required_interfaces, charm_func=None, servic
                 message = "{}, {}".format(message, charm_message)
             else:
                 message = charm_message
+    return state, message
 
-    # If the charm thinks the unit is active, check that the actual services
-    # really are active.
-    if services is not None and state == 'active':
-        # if we're passed the dict() then just grab the values as a list.
-        if isinstance(services, dict):
-            services = services.values()
-        # either extract the list of services from the dictionary, or if
-        # it is a simple string, use that. i.e. works with mixed lists.
-        _s = []
-        for s in services:
-            if isinstance(s, dict) and 'service' in s:
-                _s.append(s['service'])
-            if isinstance(s, str):
-                _s.append(s)
-        services_running = [service_running(s) for s in _s]
-        if not all(services_running):
-            not_running = [s for s, running in zip(_s, services_running)
-                           if not running]
-            message = ("Services not running that should be: {}"
-                       .format(", ".join(not_running)))
+
+def _ows_check_services_running(services, ports):
+    """Check that the services that should be running are actually running
+    and that any ports specified are being listened to.
+
+    @param services: list of strings OR dictionary specifying services/ports
+    @param ports: list of ports
+    @returns state, message: strings or None, None
+    """
+    messages = []
+    state = None
+    if services is not None:
+        services = _extract_services_list_helper(services)
+        services_running, running = _check_running_services(services)
+        if not all(running):
+            messages.append(
+                "Services not running that should be: {}"
+                .format(", ".join(_filter_tuples(services_running, False))))
             state = 'blocked'
         # also verify that the ports that should be open are open
         # NB, that ServiceManager objects only OPTIONALLY have ports
-        port_map = OrderedDict([(s['service'], s['ports'])
-                                for s in services if 'ports' in s])
-        if state == 'active' and port_map:
-            all_ports = list(itertools.chain(*port_map.values()))
-            ports_open = [port_has_listener('0.0.0.0', p)
-                          for p in all_ports]
-            if not all(ports_open):
-                not_opened = [p for p, opened in zip(all_ports, ports_open)
-                              if not opened]
-                map_not_open = OrderedDict()
-                for service, ports in port_map.items():
-                    closed_ports = set(ports).intersection(not_opened)
-                    if closed_ports:
-                        map_not_open[service] = closed_ports
-                # find which service has missing ports. They are in service
-                # order which makes it a bit easier.
-                message = (
-                    "Services with ports not open that should be: {}"
-                    .format(
-                        ", ".join([
-                            "{}: [{}]".format(
-                                service,
-                                ", ".join([str(v) for v in ports]))
-                            for service, ports in map_not_open.items()])))
-                state = 'blocked'
-
-    if ports is not None and state == 'active':
-        # and we can also check ports which we don't know the service for
-        ports_open = [port_has_listener('0.0.0.0', p) for p in ports]
+        map_not_open, ports_open = (
+            _check_listening_on_services_ports(services))
         if not all(ports_open):
-            message = (
+            # find which service has missing ports. They are in service
+            # order which makes it a bit easier.
+            message_parts = {service: ", ".join([str(v) for v in open_ports])
+                             for service, open_ports in map_not_open.items()}
+            message = ", ".join(
+                ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()])
+            messages.append(
+                "Services with ports not open that should be: {}"
+                .format(message))
+            state = 'blocked'
+
+    if ports is not None:
+        # and we can also check ports which we don't know the service for
+        ports_open, ports_open_bools = _check_listening_on_ports_list(ports)
+        if not all(ports_open_bools):
+            messages.append(
                 "Ports which should be open, but are not: {}"
-                .format(", ".join([str(p) for p, v in zip(ports, ports_open)
+                .format(", ".join([str(p) for p, v in ports_open
                                    if not v])))
             state = 'blocked'
 
-    # Set to active if all requirements have been met
-    if state == 'active':
-        message = "Unit is ready"
-        juju_log(message, "INFO")
+    if state is not None:
+        message = "; ".join(messages)
+        return state, message
 
-    status_set(state, message)
+    return None, None
+
+
+def _extract_services_list_helper(services):
+    """Extract a OrderedDict of {service: [ports]} of the supplied services
+    for use by the other functions.
+
+    The services object can either be:
+      - None : no services were passed (an empty dict is returned)
+      - a list of strings
+      - A dictionary (optionally OrderedDict) {service_name: {'service': ..}}
+      - An array of [{'service': service_name, ...}, ...]
+
+    @param services: see above
+    @returns OrderedDict(service: [ports], ...)
+    """
+    if services is None:
+        return {}
+    if isinstance(services, dict):
+        services = services.values()
+    # either extract the list of services from the dictionary, or if
+    # it is a simple string, use that. i.e. works with mixed lists.
+    _s = OrderedDict()
+    for s in services:
+        if isinstance(s, dict) and 'service' in s:
+            _s[s['service']] = s.get('ports', [])
+        if isinstance(s, str):
+            _s[s] = []
+    return _s
+
+
+def _check_running_services(services):
+    """Check that the services dict provided is actually running and provide
+    a list of (service, boolean) tuples for each service.
+
+    Returns both a zipped list of (service, boolean) and a list of booleans
+    in the same order as the services.
+
+    @param services: OrderedDict of strings: [ports], one for each service to
+                     check.
+    @returns [(service, boolean), ...], : results for checks
+             [boolean]                  : just the result of the service checks
+    """
+    services_running = [service_running(s) for s in services]
+    return list(zip(services, services_running)), services_running
+
+
+def _check_listening_on_services_ports(services, test=False):
+    """Check that the unit is actually listening (has the port open) on the
+    ports that the service specifies are open. If test is True then the
+    function returns the services with ports that are open rather than
+    closed.
+
+    Returns an OrderedDict of service: ports and a list of booleans
+
+    @param services: OrderedDict(service: [port, ...], ...)
+    @param test: default=False, if False, test for closed, otherwise open.
+    @returns OrderedDict(service: [port-not-open, ...]...), [boolean]
+    """
+    test = not(not(test))  # ensure test is True or False
+    all_ports = list(itertools.chain(*services.values()))
+    ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports]
+    map_ports = OrderedDict()
+    matched_ports = [p for p, opened in zip(all_ports, ports_states)
+                     if opened == test]  # essentially opened xor test
+    for service, ports in services.items():
+        set_ports = set(ports).intersection(matched_ports)
+        if set_ports:
+            map_ports[service] = set_ports
+    return map_ports, ports_states
+
+
+def _check_listening_on_ports_list(ports):
+    """Check that the ports list given are being listened to
+
+    Returns a list of ports being listened to and a list of the
+    booleans.
+
+    @param ports: LIST or port numbers.
+    @returns [(port_num, boolean), ...], [boolean]
+    """
+    ports_open = [port_has_listener('0.0.0.0', p) for p in ports]
+    return zip(ports, ports_open), ports_open
+
+
+def _filter_tuples(services_states, state):
+    """Return a simple list from a list of tuples according to the condition
+
+    @param services_states: LIST of (string, boolean): service and running
+           state.
+    @param state: Boolean to match the tuple against.
+    @returns [LIST of strings] that matched the tuple RHS.
+    """
+    return [s for s, b in services_states if b == state]
 
 
 def workload_state_compare(current_workload_state, workload_state):
@@ -1046,8 +1236,7 @@ def workload_state_compare(current_workload_state, workload_state):
 
 
 def incomplete_relation_data(configs, required_interfaces):
-    """
-    Check complete contexts against required_interfaces
+    """Check complete contexts against required_interfaces
     Return dictionary of incomplete relation data.
 
     configs is an OSConfigRenderer object with configs registered
@@ -1072,19 +1261,13 @@ def incomplete_relation_data(configs, required_interfaces):
               'shared-db': {'related': True}}}
     """
     complete_ctxts = configs.complete_contexts()
-    incomplete_relations = []
-    for svc_type in required_interfaces.keys():
-        # Avoid duplicates
-        found_ctxt = False
-        for interface in required_interfaces[svc_type]:
-            if interface in complete_ctxts:
-                found_ctxt = True
-        if not found_ctxt:
-            incomplete_relations.append(svc_type)
-    incomplete_context_data = {}
-    for i in incomplete_relations:
-        incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i])
-    return incomplete_context_data
+    incomplete_relations = [
+        svc_type
+        for svc_type, interfaces in required_interfaces.items()
+        if not set(interfaces).intersection(complete_ctxts)]
+    return {
+        i: configs.get_incomplete_context_data(required_interfaces[i])
+        for i in incomplete_relations}
 
 
 def do_action_openstack_upgrade(package, upgrade_callback, configs):
@@ -1145,3 +1328,245 @@ def remote_restart(rel_name, remote_service=None):
             relation_set(relation_id=rid,
                          relation_settings=trigger,
                          )
+
+
+def check_actually_paused(services=None, ports=None):
+    """Check that services listed in the services object and and ports
+    are actually closed (not listened to), to verify that the unit is
+    properly paused.
+
+    @param services: See _extract_services_list_helper
+    @returns status, : string for status (None if okay)
+             message : string for problem for status_set
+    """
+    state = None
+    message = None
+    messages = []
+    if services is not None:
+        services = _extract_services_list_helper(services)
+        services_running, services_states = _check_running_services(services)
+        if any(services_states):
+            # there shouldn't be any running so this is a problem
+            messages.append("these services running: {}"
+                            .format(", ".join(
+                                _filter_tuples(services_running, True))))
+            state = "blocked"
+        ports_open, ports_open_bools = (
+            _check_listening_on_services_ports(services, True))
+        if any(ports_open_bools):
+            message_parts = {service: ", ".join([str(v) for v in open_ports])
+                             for service, open_ports in ports_open.items()}
+            message = ", ".join(
+                ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()])
+            messages.append(
+                "these service:ports are open: {}".format(message))
+            state = 'blocked'
+    if ports is not None:
+        ports_open, bools = _check_listening_on_ports_list(ports)
+        if any(bools):
+            messages.append(
+                "these ports which should be closed, but are open: {}"
+                .format(", ".join([str(p) for p, v in ports_open if v])))
+            state = 'blocked'
+    if messages:
+        message = ("Services should be paused but {}"
+                   .format(", ".join(messages)))
+    return state, message
+
+
+def set_unit_paused():
+    """Set the unit to a paused state in the local kv() store.
+    This does NOT actually pause the unit
+    """
+    with unitdata.HookData()() as t:
+        kv = t[0]
+        kv.set('unit-paused', True)
+
+
+def clear_unit_paused():
+    """Clear the unit from a paused state in the local kv() store
+    This does NOT actually restart any services - it only clears the
+    local state.
+    """
+    with unitdata.HookData()() as t:
+        kv = t[0]
+        kv.set('unit-paused', False)
+
+
+def is_unit_paused_set():
+    """Return the state of the kv().get('unit-paused').
+    This does NOT verify that the unit really is paused.
+
+    To help with units that don't have HookData() (testing)
+    if it excepts, return False
+    """
+    try:
+        with unitdata.HookData()() as t:
+            kv = t[0]
+            # transform something truth-y into a Boolean.
+            return not(not(kv.get('unit-paused')))
+    except:
+        return False
+
+
+def pause_unit(assess_status_func, services=None, ports=None,
+               charm_func=None):
+    """Pause a unit by stopping the services and setting 'unit-paused'
+    in the local kv() store.
+
+    Also checks that the services have stopped and ports are no longer
+    being listened to.
+
+    An optional charm_func() can be called that can either raise an
+    Exception or return non None, None to indicate that the unit
+    didn't pause cleanly.
+
+    The signature for charm_func is:
+    charm_func() -> message: string
+
+    charm_func() is executed after any services are stopped, if supplied.
+
+    The services object can either be:
+      - None : no services were passed (an empty dict is returned)
+      - a list of strings
+      - A dictionary (optionally OrderedDict) {service_name: {'service': ..}}
+      - An array of [{'service': service_name, ...}, ...]
+
+    @param assess_status_func: (f() -> message: string | None) or None
+    @param services: OPTIONAL see above
+    @param ports: OPTIONAL list of port
+    @param charm_func: function to run for custom charm pausing.
+    @returns None
+    @raises Exception(message) on an error for action_fail().
+    """
+    services = _extract_services_list_helper(services)
+    messages = []
+    if services:
+        for service in services.keys():
+            stopped = service_pause(service)
+            if not stopped:
+                messages.append("{} didn't stop cleanly.".format(service))
+    if charm_func:
+        try:
+            message = charm_func()
+            if message:
+                messages.append(message)
+        except Exception as e:
+            message.append(str(e))
+    set_unit_paused()
+    if assess_status_func:
+        message = assess_status_func()
+        if message:
+            messages.append(message)
+    if messages:
+        raise Exception("Couldn't pause: {}".format("; ".join(messages)))
+
+
+def resume_unit(assess_status_func, services=None, ports=None,
+                charm_func=None):
+    """Resume a unit by starting the services and clearning 'unit-paused'
+    in the local kv() store.
+
+    Also checks that the services have started and ports are being listened to.
+
+    An optional charm_func() can be called that can either raise an
+    Exception or return non None to indicate that the unit
+    didn't resume cleanly.
+
+    The signature for charm_func is:
+    charm_func() -> message: string
+
+    charm_func() is executed after any services are started, if supplied.
+
+    The services object can either be:
+      - None : no services were passed (an empty dict is returned)
+      - a list of strings
+      - A dictionary (optionally OrderedDict) {service_name: {'service': ..}}
+      - An array of [{'service': service_name, ...}, ...]
+
+    @param assess_status_func: (f() -> message: string | None) or None
+    @param services: OPTIONAL see above
+    @param ports: OPTIONAL list of port
+    @param charm_func: function to run for custom charm resuming.
+    @returns None
+    @raises Exception(message) on an error for action_fail().
+    """
+    services = _extract_services_list_helper(services)
+    messages = []
+    if services:
+        for service in services.keys():
+            started = service_resume(service)
+            if not started:
+                messages.append("{} didn't start cleanly.".format(service))
+    if charm_func:
+        try:
+            message = charm_func()
+            if message:
+                messages.append(message)
+        except Exception as e:
+            message.append(str(e))
+    clear_unit_paused()
+    if assess_status_func:
+        message = assess_status_func()
+        if message:
+            messages.append(message)
+    if messages:
+        raise Exception("Couldn't resume: {}".format("; ".join(messages)))
+
+
+def make_assess_status_func(*args, **kwargs):
+    """Creates an assess_status_func() suitable for handing to pause_unit()
+    and resume_unit().
+
+    This uses the _determine_os_workload_status(...) function to determine
+    what the workload_status should be for the unit.  If the unit is
+    not in maintenance or active states, then the message is returned to
+    the caller.  This is so an action that doesn't result in either a
+    complete pause or complete resume can signal failure with an action_fail()
+    """
+    def _assess_status_func():
+        state, message = _determine_os_workload_status(*args, **kwargs)
+        status_set(state, message)
+        if state not in ['maintenance', 'active']:
+            return message
+        return None
+
+    return _assess_status_func
+
+
+def pausable_restart_on_change(restart_map, stopstart=False):
+    """A restart_on_change decorator that checks to see if the unit is
+    paused. If it is paused then the decorated function doesn't fire.
+
+    This is provided as a helper, as the @restart_on_change(...) decorator
+    is in core.host, yet the openstack specific helpers are in this file
+    (contrib.openstack.utils).  Thus, this needs to be an optional feature
+    for openstack charms (or charms that wish to use the openstack
+    pause/resume type features).
+
+    It is used as follows:
+
+        from contrib.openstack.utils import (
+            pausable_restart_on_change as restart_on_change)
+
+        @restart_on_change(restart_map, stopstart=<boolean>)
+        def some_hook(...):
+            pass
+
+    see core.utils.restart_on_change() for more details.
+
+    @param f: the function to decorate
+    @param restart_map: the restart map {conf_file: [services]}
+    @param stopstart: DEFAULT false; whether to stop, start or just restart
+    @returns decorator to use a restart_on_change with pausability
+    """
+    def wrap(f):
+        @functools.wraps(f)
+        def wrapped_f(*args, **kwargs):
+            if is_unit_paused_set():
+                return f(*args, **kwargs)
+            # otherwise, normal restart_on_change functionality
+            return restart_on_change_helper(
+                (lambda: f(*args, **kwargs)), restart_map, stopstart)
+        return wrapped_f
+    return wrap

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -24,6 +24,8 @@
 #  Adam Gandelman <adamg@ubuntu.com>
 #
 import bisect
+import errno
+import hashlib
 import six
 
 import os
@@ -163,7 +165,7 @@ class Pool(object):
         :return: None
         """
         # read-only is easy, writeback is much harder
-        mode = get_cache_mode(cache_pool)
+        mode = get_cache_mode(self.service, cache_pool)
         if mode == 'readonly':
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none'])
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
@@ -171,7 +173,7 @@ class Pool(object):
         elif mode == 'writeback':
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'forward'])
             # Flush the cache and wait for it to return
-            check_call(['ceph', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all'])
+            check_call(['rados', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all'])
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name])
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
 
@@ -259,6 +261,134 @@ class ErasurePool(Pool):
        Returns json formatted output"""
 
 
+def get_mon_map(service):
+    """
+    Returns the current monitor map.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :return: json string. :raise: ValueError if the monmap fails to parse.
+      Also raises CalledProcessError if our ceph command fails
+    """
+    try:
+        mon_status = check_output(
+            ['ceph', '--id', service,
+             'mon_status', '--format=json'])
+        try:
+            return json.loads(mon_status)
+        except ValueError as v:
+            log("Unable to parse mon_status json: {}. Error: {}".format(
+                mon_status, v.message))
+            raise
+    except CalledProcessError as e:
+        log("mon_status command failed with message: {}".format(
+            e.message))
+        raise
+
+
+def hash_monitor_names(service):
+    """
+    Uses the get_mon_map() function to get information about the monitor
+    cluster.
+    Hash the name of each monitor.  Return a sorted list of monitor hashes
+    in an ascending order.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :rtype : dict.   json dict of monitor name, ip address and rank
+    example: {
+        'name': 'ip-172-31-13-165',
+        'rank': 0,
+        'addr': '172.31.13.165:6789/0'}
+    """
+    try:
+        hash_list = []
+        monitor_list = get_mon_map(service=service)
+        if monitor_list['monmap']['mons']:
+            for mon in monitor_list['monmap']['mons']:
+                hash_list.append(
+                    hashlib.sha224(mon['name'].encode('utf-8')).hexdigest())
+            return sorted(hash_list)
+        else:
+            return None
+    except (ValueError, CalledProcessError):
+        raise
+
+
+def monitor_key_delete(service, key):
+    """
+    Delete a key and value pair from the monitor cluster
+    :param service: six.string_types. The Ceph user name to run the command under
+    Deletes a key value pair on the monitor cluster.
+    :param key: six.string_types.  The key to delete.
+    """
+    try:
+        check_output(
+            ['ceph', '--id', service,
+             'config-key', 'del', str(key)])
+    except CalledProcessError as e:
+        log("Monitor config-key put failed with message: {}".format(
+            e.output))
+        raise
+
+
+def monitor_key_set(service, key, value):
+    """
+    Sets a key value pair on the monitor cluster.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param key: six.string_types.  The key to set.
+    :param value: The value to set.  This will be converted to a string
+        before setting
+    """
+    try:
+        check_output(
+            ['ceph', '--id', service,
+             'config-key', 'put', str(key), str(value)])
+    except CalledProcessError as e:
+        log("Monitor config-key put failed with message: {}".format(
+            e.output))
+        raise
+
+
+def monitor_key_get(service, key):
+    """
+    Gets the value of an existing key in the monitor cluster.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param key: six.string_types.  The key to search for.
+    :return: Returns the value of that key or None if not found.
+    """
+    try:
+        output = check_output(
+            ['ceph', '--id', service,
+             'config-key', 'get', str(key)])
+        return output
+    except CalledProcessError as e:
+        log("Monitor config-key get failed with message: {}".format(
+            e.output))
+        return None
+
+
+def monitor_key_exists(service, key):
+    """
+    Searches for the existence of a key in the monitor cluster.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param key: six.string_types.  The key to search for
+    :return: Returns True if the key exists, False if not and raises an
+     exception if an unknown error occurs. :raise: CalledProcessError if
+     an unknown error occurs
+    """
+    try:
+        check_call(
+            ['ceph', '--id', service,
+             'config-key', 'exists', str(key)])
+        # I can return true here regardless because Ceph returns
+        # ENOENT if the key wasn't found
+        return True
+    except CalledProcessError as e:
+        if e.returncode == errno.ENOENT:
+            return False
+        else:
+            log("Unknown error from ceph config-get exists: {} {}".format(
+                e.returncode, e.output))
+            raise
+
+
 def get_erasure_profile(service, name):
     """
     :param service: six.string_types. The Ceph user name to run the command under

hooks/charmhelpers/core/hookenv.py

@@ -912,6 +912,24 @@ def payload_status_set(klass, pid, status):
     subprocess.check_call(cmd)
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def resource_get(name):
+    """used to fetch the resource path of the given name.
+
+    <name> must match a name of defined resource in metadata.yaml
+
+    returns either a path or False if resource not available
+    """
+    if not name:
+        return False
+
+    cmd = ['resource-get', name]
+    try:
+        return subprocess.check_output(cmd).decode('UTF-8')
+    except subprocess.CalledProcessError:
+        return False
+
+
 @cached
 def juju_version():
     """Full version string (eg. '1.23.3.1-trusty-amd64')"""
@@ -976,3 +994,16 @@ def _run_atexit():
     for callback, args, kwargs in reversed(_atexit):
         callback(*args, **kwargs)
     del _atexit[:]
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get_primary_address(binding):
+    '''
+    Retrieve the primary network address for a named binding
+
+    :param binding: string. The name of a relation of extra-binding
+    :return: string. The primary IP address for the named binding
+    :raise: NotImplementedError if run on Juju < 2.0
+    '''
+    cmd = ['network-get', '--primary-address', binding]
+    return subprocess.check_output(cmd).strip()

hooks/charmhelpers/core/host.py

@@ -30,6 +30,8 @@ import random
 import string
 import subprocess
 import hashlib
+import functools
+import itertools
 from contextlib import contextmanager
 from collections import OrderedDict
 
@@ -428,27 +430,47 @@ def restart_on_change(restart_map, stopstart=False):
     restarted if any file matching the pattern got changed, created
     or removed. Standard wildcards are supported, see documentation
     for the 'glob' module for more information.
+
+    @param restart_map: {path_file_name: [service_name, ...]
+    @param stopstart: DEFAULT false; whether to stop, start OR restart
+    @returns result from decorated function
     """
     def wrap(f):
+        @functools.wraps(f)
         def wrapped_f(*args, **kwargs):
-            checksums = {path: path_hash(path) for path in restart_map}
-            f(*args, **kwargs)
-            restarts = []
-            for path in restart_map:
-                if path_hash(path) != checksums[path]:
-                    restarts += restart_map[path]
-            services_list = list(OrderedDict.fromkeys(restarts))
-            if not stopstart:
-                for service_name in services_list:
-                    service('restart', service_name)
-            else:
-                for action in ['stop', 'start']:
-                    for service_name in services_list:
-                        service(action, service_name)
+            return restart_on_change_helper(
+                (lambda: f(*args, **kwargs)), restart_map, stopstart)
         return wrapped_f
     return wrap
 
 
+def restart_on_change_helper(lambda_f, restart_map, stopstart=False):
+    """Helper function to perform the restart_on_change function.
+
+    This is provided for decorators to restart services if files described
+    in the restart_map have changed after an invocation of lambda_f().
+
+    @param lambda_f: function to call.
+    @param restart_map: {file: [service, ...]}
+    @param stopstart: whether to stop, start or restart a service
+    @returns result of lambda_f()
+    """
+    checksums = {path: path_hash(path) for path in restart_map}
+    r = lambda_f()
+    # create a list of lists of the services to restart
+    restarts = [restart_map[path]
+                for path in restart_map
+                if path_hash(path) != checksums[path]]
+    # create a flat list of ordered services without duplicates from lists
+    services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts)))
+    if services_list:
+        actions = ('stop', 'start') if stopstart else ('restart',)
+        for action in actions:
+            for service_name in services_list:
+                service(action, service_name)
+    return r
+
+
 def lsb_release():
     """Return /etc/lsb-release in a dict"""
     d = {}

hooks/neutron_hooks.py

@@ -40,6 +40,7 @@ from charmhelpers.payload.execd import execd_preinstall
 from charmhelpers.core.sysctl import create as create_sysctl
 
 from charmhelpers.contrib.charmsupport import nrpe
+from charmhelpers.contrib.hardening.harden import harden
 
 import sys
 from neutron_utils import (
@@ -74,6 +75,7 @@ CONFIGS = register_configs()
 
 
 @hooks.hook('install.real')
+@harden()
 def install():
     status_set('maintenance', 'Executing pre-install')
     execd_preinstall()
@@ -104,6 +106,7 @@ def install():
 
 @hooks.hook('config-changed')
 @restart_on_change(restart_map())
+@harden()
 def config_changed():
     global CONFIGS
     if git_install_requested():
@@ -151,6 +154,7 @@ def config_changed():
 
 
 @hooks.hook('upgrade-charm')
+@harden()
 def upgrade_charm():
     install()
     config_changed()
@@ -319,6 +323,12 @@ def ha_relation_destroyed():
         remove_legacy_ha_files()
 
 
+@hooks.hook('update-status')
+@harden()
+def update_status():
+    log('Updating status.')
+
+
 if __name__ == '__main__':
     try:
         hooks.execute(sys.argv)

hooks/update-status

@@ -0,0 +1 @@
+neutron_hooks.py

tests/basic_deployment.py

@@ -595,33 +595,49 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment):
             return
 
         unit = self.neutron_gateway_sentry
-        conf = '/etc/neutron/plugins/ml2/ml2_conf.ini'
-
-        expected = {
-            'ml2': {
-                'type_drivers': 'gre,vxlan,vlan,flat',
-                'tenant_network_types': 'gre,vxlan,vlan,flat',
-                'mechanism_drivers': 'openvswitch,hyperv,l2population'
-            },
-            'ml2_type_gre': {
-                'tunnel_id_ranges': '1:1000'
-            },
-            'ml2_type_vxlan': {
-                'vni_ranges': '1001:2000'
-            },
-            'ovs': {
-                'enable_tunneling': 'True',
-                'local_ip': self.get_private_address(unit)
-            },
-            'agent': {
-                'tunnel_types': 'gre',
-                'l2_population': 'False'
-            },
-            'securitygroup': {
-                'firewall_driver': 'neutron.agent.linux.iptables_firewall.'
-                                   'OVSHybridIptablesFirewallDriver'
+        if self._get_openstack_release() < self.trusty_mitaka:
+            conf = '/etc/neutron/plugins/ml2/ml2_conf.ini'
+            expected = {
+                'ml2': {
+                    'type_drivers': 'gre,vxlan,vlan,flat',
+                    'tenant_network_types': 'gre,vxlan,vlan,flat',
+                    'mechanism_drivers': 'openvswitch,hyperv,l2population'
+                },
+                'ml2_type_gre': {
+                    'tunnel_id_ranges': '1:1000'
+                },
+                'ml2_type_vxlan': {
+                    'vni_ranges': '1001:2000'
+                },
+                'ovs': {
+                    'enable_tunneling': 'True',
+                    'local_ip': self.get_private_address(unit)
+                },
+                'agent': {
+                    'tunnel_types': 'gre',
+                    'l2_population': 'False'
+                },
+                'securitygroup': {
+                    'firewall_driver': 'neutron.agent.linux.iptables_firewall.'
+                                       'OVSHybridIptablesFirewallDriver'
+                }
+            }
+        else:
+            conf = '/etc/neutron/plugins/ml2/openvswitch_agent.ini'
+            expected = {
+                'ovs': {
+                    'enable_tunneling': 'True',
+                    'local_ip': self.get_private_address(unit)
+                },
+                'agent': {
+                    'tunnel_types': 'gre',
+                    'l2_population': 'False'
+                },
+                'securitygroup': {
+                    'firewall_driver': 'neutron.agent.linux.iptables_firewall.'
+                                       'OVSHybridIptablesFirewallDriver'
+                }
             }
-        }
 
         for section, pairs in expected.iteritems():
             ret = u.validate_config_data(unit, conf, section, pairs)

tests/charmhelpers/contrib/amulet/utils.py

@@ -782,15 +782,20 @@ class AmuletUtils(object):
 
 # amulet juju action helpers:
     def run_action(self, unit_sentry, action,
-                   _check_output=subprocess.check_output):
+                   _check_output=subprocess.check_output,
+                   params=None):
         """Run the named action on a given unit sentry.
 
+        params a dict of parameters to use
         _check_output parameter is used for dependency injection.
 
         @return action_id.
         """
         unit_id = unit_sentry.info["unit_name"]
         command = ["juju", "action", "do", "--format=json", unit_id, action]
+        if params is not None:
+            for key, value in params.iteritems():
+                command.append("{}={}".format(key, value))
         self.log.info("Running command: %s\n" % " ".join(command))
         output = _check_output(command, universal_newlines=True)
         data = json.loads(output)

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -27,7 +27,11 @@ import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
-import novaclient.v1_1.client as nova_client
+from keystoneclient.auth.identity import v3 as keystone_id_v3
+from keystoneclient import session as keystone_session
+from keystoneclient.v3 import client as keystone_client_v3
+
+import novaclient.client as nova_client
 import pika
 import swiftclient
 
@@ -38,6 +42,8 @@ from charmhelpers.contrib.amulet.utils import (
 DEBUG = logging.DEBUG
 ERROR = logging.ERROR
 
+NOVA_CLIENT_VERSION = "2"
+
 
 class OpenStackAmuletUtils(AmuletUtils):
     """OpenStack amulet utilities.
@@ -139,7 +145,7 @@ class OpenStackAmuletUtils(AmuletUtils):
                 return "role {} does not exist".format(e['name'])
         return ret
 
-    def validate_user_data(self, expected, actual):
+    def validate_user_data(self, expected, actual, api_version=None):
         """Validate user data.
 
            Validate a list of actual user data vs a list of expected user
@@ -150,10 +156,15 @@ class OpenStackAmuletUtils(AmuletUtils):
         for e in expected:
             found = False
             for act in actual:
-                a = {'enabled': act.enabled, 'name': act.name,
-                     'email': act.email, 'tenantId': act.tenantId,
-                     'id': act.id}
-                if e['name'] == a['name']:
+                if e['name'] == act.name:
+                    a = {'enabled': act.enabled, 'name': act.name,
+                         'email': act.email, 'id': act.id}
+                    if api_version == 3:
+                        a['default_project_id'] = getattr(act,
+                                                          'default_project_id',
+                                                          'none')
+                    else:
+                        a['tenantId'] = act.tenantId
                     found = True
                     ret = self._validate_dict_data(e, a)
                     if ret:
@@ -188,15 +199,30 @@ class OpenStackAmuletUtils(AmuletUtils):
         return cinder_client.Client(username, password, tenant, ept)
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
-                                    tenant):
+                                    tenant=None, api_version=None,
+                                    keystone_ip=None):
         """Authenticates admin user with the keystone admin endpoint."""
         self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
-        service_ip = unit.relation('shared-db',
-                                   'mysql:shared-db')['private-address']
-        ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
-        return keystone_client.Client(username=user, password=password,
-                                      tenant_name=tenant, auth_url=ep)
+        if not keystone_ip:
+            keystone_ip = unit.relation('shared-db',
+                                        'mysql:shared-db')['private-address']
+        base_ep = "http://{}:35357".format(keystone_ip.strip().decode('utf-8'))
+        if not api_version or api_version == 2:
+            ep = base_ep + "/v2.0"
+            return keystone_client.Client(username=user, password=password,
+                                          tenant_name=tenant, auth_url=ep)
+        else:
+            ep = base_ep + "/v3"
+            auth = keystone_id_v3.Password(
+                user_domain_name='admin_domain',
+                username=user,
+                password=password,
+                domain_name='admin_domain',
+                auth_url=ep,
+            )
+            sess = keystone_session.Session(auth=auth)
+            return keystone_client_v3.Client(session=sess)
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
@@ -225,7 +251,8 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return nova_client.Client(username=user, api_key=password,
+        return nova_client.Client(NOVA_CLIENT_VERSION,
+                                  username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
     def authenticate_swift_user(self, keystone, user, password, tenant):

unit_tests/test_actions_git_reinstall.py

@@ -1,26 +1,23 @@
+import sys
+
 from mock import patch, MagicMock
+from test_utils import CharmTestCase
 
-with patch('charmhelpers.core.hookenv.config') as config:
-    config.return_value = 'neutron'
-    import neutron_utils as utils  # noqa
+# python-apt is not installed as part of test-requirements but is imported by
+# some charmhelpers modules so create a fake import.
+sys.modules['apt'] = MagicMock()
+sys.modules['apt_pkg'] = MagicMock()
 
-from test_utils import (
-    CharmTestCase
-)
-
-# Need to do some early patching to get the module loaded.
-_register_configs = utils.register_configs
-_restart_map = utils.restart_map
-
-utils.register_configs = MagicMock()
-utils.restart_map = MagicMock()
-
-with patch('charmhelpers.core.hookenv.status_set'):
-    import git_reinstall
-
-# Unpatch it now that its loaded.
-utils.register_configs = _register_configs
-utils.restart_map = _restart_map
+with patch('charmhelpers.core.hookenv.config'):
+    with patch('neutron_utils.restart_map'):
+        with patch('neutron_utils.register_configs'):
+            with patch('charmhelpers.contrib.hardening.harden.harden') as \
+                    mock_dec:
+                mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f:
+                                        lambda *args, **kwargs:
+                                            f(*args, **kwargs))
+                with patch('charmhelpers.core.hookenv.status_set'):
+                    import git_reinstall
 
 TO_PATCH = [
     'config',

unit_tests/test_neutron_hooks.py

@@ -1,19 +1,22 @@
-from mock import MagicMock, patch, call
+import sys
+
 import yaml
+
+from mock import MagicMock, patch, call
+
+# python-apt is not installed as part of test-requirements but is imported by
+# some charmhelpers modules so create a fake import.
+sys.modules['apt'] = MagicMock()
+sys.modules['apt_pkg'] = MagicMock()
+
 import charmhelpers.core.hookenv as hookenv
-hookenv.config = MagicMock()
-import neutron_utils as utils
-_register_configs = utils.register_configs
-_restart_map = utils.restart_map
-utils.register_configs = MagicMock()
-utils.restart_map = MagicMock()
-
-with patch('charmhelpers.core.hookenv.status_set'):
+with patch('charmhelpers.contrib.hardening.harden.harden') as \
+        mock_dec:
+    mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f:
+                            lambda *args, **kwargs:
+                                f(*args, **kwargs))
     import neutron_hooks as hooks
 
-utils.register_configs = _register_configs
-utils.restart_map = _restart_map
-
 from test_utils import CharmTestCase
 
 
@@ -54,10 +57,6 @@ TO_PATCH = [
 ]
 
 
-def passthrough(value):
-    return value
-
-
 class TestQuantumHooks(CharmTestCase):
 
     def setUp(self):
@@ -66,7 +65,8 @@ class TestQuantumHooks(CharmTestCase):
         self.test_config.set('openstack-origin', 'cloud:precise-havana')
         self.test_config.set('plugin', 'ovs')
         self.lsb_release.return_value = {'DISTRIB_CODENAME': 'precise'}
-        self.b64decode.side_effect = passthrough
+        # passthrough
+        self.b64decode.side_effect = lambda arg: arg
         hookenv.config.side_effect = self.test_config.get
         hooks.hooks._config_save = False
 
@@ -78,7 +78,7 @@ class TestQuantumHooks(CharmTestCase):
         self.valid_plugin.return_value = True
         _pkgs = ['foo', 'bar']
         self.filter_installed_packages.return_value = _pkgs
-        self._call_hook('install')
+        self._call_hook('install.real')
         self.configure_installation_source.assert_called_with(
             'cloud:precise-havana'
         )
@@ -93,7 +93,7 @@ class TestQuantumHooks(CharmTestCase):
 
     def test_install_hook_precise_nocloudarchive(self):
         self.test_config.set('openstack-origin', 'distro')
-        self._call_hook('install')
+        self._call_hook('install.real')
         self.configure_installation_source.assert_called_with(
             'cloud:precise-icehouse'
         )
@@ -101,11 +101,11 @@ class TestQuantumHooks(CharmTestCase):
     @patch('sys.exit')
     def test_install_hook_invalid_plugin(self, _exit):
         self.valid_plugin.return_value = False
-        self._call_hook('install')
+        self._call_hook('install.real')
         self.assertTrue(self.log.called)
         _exit.assert_called_with(1)
 
-    @patch.object(utils, 'git_install_requested')
+    @patch('neutron_utils.git_install_requested')
     def test_install_hook_git(self, git_requested):
         git_requested.return_value = True
         self.valid_plugin.return_value = True
@@ -126,7 +126,7 @@ class TestQuantumHooks(CharmTestCase):
         projects_yaml = yaml.dump(openstack_origin_git)
         self.test_config.set('openstack-origin', repo)
         self.test_config.set('openstack-origin-git', projects_yaml)
-        self._call_hook('install')
+        self._call_hook('install.real')
         self.configure_installation_source.assert_called_with(
             'cloud:trusty-juno'
         )

unit_tests/test_neutron_utils.py

@@ -1,12 +1,9 @@
-from mock import MagicMock, call, patch, ANY
 import collections
-import charmhelpers.contrib.openstack.templating as templating
 
-templating.OSConfigRenderer = MagicMock()
+from mock import MagicMock, call, patch, ANY
 
+import charmhelpers.core.hookenv as hookenv
 import neutron_utils
-
-
 try:
     import neutronclient
 except ImportError:
@@ -16,9 +13,6 @@ from test_utils import (
     CharmTestCase
 )
 
-import charmhelpers.core.hookenv as hookenv
-
-
 TO_PATCH = [
     'config',
     'get_os_codename_install_source',
@@ -248,8 +242,9 @@ class TestNeutronUtils(CharmTestCase):
                  call('br1', 'eth0.200', promisc=True)]
         self.add_bridge_port.assert_has_calls(calls)
 
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
     @patch.object(neutron_utils, 'git_install_requested')
-    def test_do_openstack_upgrade(self, git_requested):
+    def test_do_openstack_upgrade(self, git_requested, mock_renderer):
         git_requested.return_value = False
         self.config.side_effect = self.test_config.get
         self.is_relation_made.return_value = False
@@ -271,7 +266,8 @@ class TestNeutronUtils(CharmTestCase):
             'cloud:precise-havana'
         )
 
-    def test_register_configs_ovs(self):
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
+    def test_register_configs_ovs(self, mock_renderer):
         self.config.return_value = 'ovs'
         self.is_relation_made.return_value = False
         configs = neutron_utils.register_configs()
@@ -285,7 +281,8 @@ class TestNeutronUtils(CharmTestCase):
         for conf in confs:
             configs.register.assert_any_call(conf, ANY)
 
-    def test_register_configs_ovs_odl(self):
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
+    def test_register_configs_ovs_odl(self, mock_renderer):
         self.config.side_effect = self.test_config.get
         self.test_config.set('plugin', 'ovs-odl')
         self.is_relation_made.return_value = False
@@ -300,7 +297,8 @@ class TestNeutronUtils(CharmTestCase):
         for conf in confs:
             configs.register.assert_any_call(conf, ANY)
 
-    def test_register_configs_amqp_nova(self):
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
+    def test_register_configs_amqp_nova(self, mock_renderer):
         self.config.return_value = 'ovs'
         self.is_relation_made.return_value = True
         configs = neutron_utils.register_configs()
@@ -420,7 +418,8 @@ class TestNeutronUtils(CharmTestCase):
 
         self.assertDictEqual(neutron_utils.restart_map(), ex_map)
 
-    def test_register_configs_nsx(self):
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
+    def test_register_configs_nsx(self, mock_renderer):
         self.config.return_value = 'nsx'
         configs = neutron_utils.register_configs()
         confs = [neutron_utils.NEUTRON_DHCP_AGENT_CONF,
@@ -443,7 +442,8 @@ class TestNeutronUtils(CharmTestCase):
             any_order=True,
         )
 
-    def test_register_configs_pre_install(self):
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
+    def test_register_configs_pre_install(self, mock_renderer):
         self.config.return_value = 'ovs'
         self.is_relation_made.return_value = False
         configs = neutron_utils.register_configs()

unit_tests/test_utils.py

@@ -1,6 +1,7 @@
-import logging
-import unittest
 import os
+import logging
+
+import unittest
 import yaml
 
 from mock import patch