Resync charmhelpers for fixes to add_source

Closes-Bug: 1370933

charm-helpers-sync.yaml

@@ -3,6 +3,7 @@ destination: charmhelpers
 include:
     - core
     - fetch
+    - osplatform
     - payload
     - contrib.openstack|inc=*
     - contrib.charmsupport

charmhelpers/__init__.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 # Bootstrap charm-helpers, installing its dependencies if necessary using
 # only standard libraries.

charmhelpers/contrib/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/charmsupport/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/charmsupport/nrpe.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 """Compatibility with the nrpe-external-master charm"""
 # Copyright 2012 Canonical Ltd.
@@ -40,6 +38,7 @@ from charmhelpers.core.hookenv import (
 )
 
 from charmhelpers.core.host import service
+from charmhelpers.core import host
 
 # This module adds compatibility with the nrpe-external-master and plain nrpe
 # subordinate charms. To use it in your charm:
@@ -110,6 +109,13 @@ from charmhelpers.core.host import service
 #    def local_monitors_relation_changed():
 #        update_nrpe_config()
 #
+# 4.a If your charm is a subordinate charm set primary=False
+#
+#    from charmsupport.nrpe import NRPE
+#    (...)
+#    def update_nrpe_config():
+#        nrpe_compat = NRPE(primary=False)
+#
 # 5. ln -s hooks.py nrpe-external-master-relation-changed
 #    ln -s hooks.py local-monitors-relation-changed
 
@@ -148,6 +154,13 @@ define service {{
         self.description = description
         self.check_cmd = self._locate_cmd(check_cmd)
 
+    def _get_check_filename(self):
+        return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command))
+
+    def _get_service_filename(self, hostname):
+        return os.path.join(NRPE.nagios_exportdir,
+                            'service__{}_{}.cfg'.format(hostname, self.command))
+
     def _locate_cmd(self, check_cmd):
         search_path = (
             '/usr/lib/nagios/plugins',
@@ -163,9 +176,21 @@ define service {{
         log('Check command not found: {}'.format(parts[0]))
         return ''
 
+    def _remove_service_files(self):
+        if not os.path.exists(NRPE.nagios_exportdir):
+            return
+        for f in os.listdir(NRPE.nagios_exportdir):
+            if f.endswith('_{}.cfg'.format(self.command)):
+                os.remove(os.path.join(NRPE.nagios_exportdir, f))
+
+    def remove(self, hostname):
+        nrpe_check_file = self._get_check_filename()
+        if os.path.exists(nrpe_check_file):
+            os.remove(nrpe_check_file)
+        self._remove_service_files()
+
     def write(self, nagios_context, hostname, nagios_servicegroups):
-        nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
+        nrpe_check_file = self._get_check_filename()
-            self.command)
         with open(nrpe_check_file, 'w') as nrpe_check_config:
             nrpe_check_config.write("# check {}\n".format(self.shortname))
             nrpe_check_config.write("command[{}]={}\n".format(
@@ -180,9 +205,7 @@ define service {{
 
     def write_service_config(self, nagios_context, hostname,
                              nagios_servicegroups):
-        for f in os.listdir(NRPE.nagios_exportdir):
+        self._remove_service_files()
-            if re.search('.*{}.cfg'.format(self.command), f):
-                os.remove(os.path.join(NRPE.nagios_exportdir, f))
 
         templ_vars = {
             'nagios_hostname': hostname,
@@ -192,8 +215,7 @@ define service {{
             'command': self.command,
         }
         nrpe_service_text = Check.service_template.format(**templ_vars)
-        nrpe_service_file = '{}/service__{}_{}.cfg'.format(
+        nrpe_service_file = self._get_service_filename(hostname)
-            NRPE.nagios_exportdir, hostname, self.command)
         with open(nrpe_service_file, 'w') as nrpe_service_config:
             nrpe_service_config.write(str(nrpe_service_text))
 
@@ -205,10 +227,12 @@ class NRPE(object):
     nagios_logdir = '/var/log/nagios'
     nagios_exportdir = '/var/lib/nagios/export'
     nrpe_confdir = '/etc/nagios/nrpe.d'
+    homedir = '/var/lib/nagios'  # home dir provided by nagios-nrpe-server
 
-    def __init__(self, hostname=None):
+    def __init__(self, hostname=None, primary=True):
         super(NRPE, self).__init__()
         self.config = config()
+        self.primary = primary
         self.nagios_context = self.config['nagios_context']
         if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']:
             self.nagios_servicegroups = self.config['nagios_servicegroups']
@@ -218,12 +242,38 @@ class NRPE(object):
         if hostname:
             self.hostname = hostname
         else:
-            self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
+            nagios_hostname = get_nagios_hostname()
+            if nagios_hostname:
+                self.hostname = nagios_hostname
+            else:
+                self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
         self.checks = []
+        # Iff in an nrpe-external-master relation hook, set primary status
+        relation = relation_ids('nrpe-external-master')
+        if relation:
+            log("Setting charm primary status {}".format(primary))
+            for rid in relation_ids('nrpe-external-master'):
+                relation_set(relation_id=rid, relation_settings={'primary': self.primary})
 
     def add_check(self, *args, **kwargs):
         self.checks.append(Check(*args, **kwargs))
 
+    def remove_check(self, *args, **kwargs):
+        if kwargs.get('shortname') is None:
+            raise ValueError('shortname of check must be specified')
+
+        # Use sensible defaults if they're not specified - these are not
+        # actually used during removal, but they're required for constructing
+        # the Check object; check_disk is chosen because it's part of the
+        # nagios-plugins-basic package.
+        if kwargs.get('check_cmd') is None:
+            kwargs['check_cmd'] = 'check_disk'
+        if kwargs.get('description') is None:
+            kwargs['description'] = ''
+
+        check = Check(*args, **kwargs)
+        check.remove(self.hostname)
+
     def write(self):
         try:
             nagios_uid = pwd.getpwnam('nagios').pw_uid
@@ -260,7 +310,7 @@ def get_nagios_hostcontext(relation_name='nrpe-external-master'):
     :param str relation_name: Name of relation nrpe sub joined to
     """
     for rel in relations_of_type(relation_name):
-        if 'nagios_hostname' in rel:
+        if 'nagios_host_context' in rel:
             return rel['nagios_host_context']
 
 
@@ -289,18 +339,30 @@ def get_nagios_unit_name(relation_name='nrpe-external-master'):
     return unit
 
 
-def add_init_service_checks(nrpe, services, unit_name):
+def add_init_service_checks(nrpe, services, unit_name, immediate_check=True):
     """
     Add checks for each service in list
 
     :param NRPE nrpe: NRPE object to add check to
     :param list services: List of services to check
     :param str unit_name: Unit name to use in check description
+    :param bool immediate_check: For sysv init, run the service check immediately
     """
     for svc in services:
+        # Don't add a check for these services from neutron-gateway
+        if svc in ['ext-port', 'os-charm-phy-nic-mtu']:
+            next
+
         upstart_init = '/etc/init/%s.conf' % svc
         sysv_init = '/etc/init.d/%s' % svc
-        if os.path.exists(upstart_init):
+
+        if host.init_is_systemd():
+            nrpe.add_check(
+                shortname=svc,
+                description='process check {%s}' % unit_name,
+                check_cmd='check_systemd.py %s' % svc
+            )
+        elif os.path.exists(upstart_init):
             nrpe.add_check(
                 shortname=svc,
                 description='process check {%s}' % unit_name,
@@ -308,21 +370,31 @@ def add_init_service_checks(nrpe, services, unit_name):
             )
         elif os.path.exists(sysv_init):
             cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
-            cron_file = ('*/5 * * * * root '
+            checkpath = '%s/service-check-%s.txt' % (nrpe.homedir, svc)
-                         '/usr/local/lib/nagios/plugins/check_exit_status.pl '
+            croncmd = (
-                         '-s /etc/init.d/%s status > '
+                '/usr/local/lib/nagios/plugins/check_exit_status.pl '
-                         '/var/lib/nagios/service-check-%s.txt\n' % (svc,
+                '-e -s /etc/init.d/%s status' % svc
-                                                                     svc)
+            )
-                         )
+            cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath)
             f = open(cronpath, 'w')
             f.write(cron_file)
             f.close()
             nrpe.add_check(
                 shortname=svc,
-                description='process check {%s}' % unit_name,
+                description='service check {%s}' % unit_name,
-                check_cmd='check_status_file.py -f '
+                check_cmd='check_status_file.py -f %s' % checkpath,
-                          '/var/lib/nagios/service-check-%s.txt' % svc,
             )
+            # if /var/lib/nagios doesn't exist open(checkpath, 'w') will fail
+            # (LP: #1670223).
+            if immediate_check and os.path.isdir(nrpe.homedir):
+                f = open(checkpath, 'w')
+                subprocess.call(
+                    croncmd.split(),
+                    stdout=f,
+                    stderr=subprocess.STDOUT
+                )
+                f.close()
+                os.chmod(checkpath, 0o644)
 
 
 def copy_nrpe_checks():

charmhelpers/contrib/charmsupport/volumes.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 '''
 Functions for managing volumes in juju units. One volume is supported per unit.

charmhelpers/contrib/hahelpers/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/hahelpers/apache.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 #
 # Copyright 2012 Canonical Ltd.
@@ -24,6 +22,7 @@
 #  Adam Gandelman <adamg@ubuntu.com>
 #
 
+import os
 import subprocess
 
 from charmhelpers.core.hookenv import (
@@ -74,9 +73,23 @@ def get_ca_cert():
     return ca_cert
 
 
+def retrieve_ca_cert(cert_file):
+    cert = None
+    if os.path.isfile(cert_file):
+        with open(cert_file, 'r') as crt:
+            cert = crt.read()
+    return cert
+
+
 def install_ca_cert(ca_cert):
     if ca_cert:
-        with open('/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt',
+        cert_file = ('/usr/local/share/ca-certificates/'
-                  'w') as crt:
+                     'keystone_juju_ca_cert.crt')
-            crt.write(ca_cert)
+        old_cert = retrieve_ca_cert(cert_file)
-        subprocess.check_call(['update-ca-certificates', '--fresh'])
+        if old_cert and old_cert == ca_cert:
+            log("CA cert is the same as installed version", level=INFO)
+        else:
+            log("Installing new CA cert", level=INFO)
+            with open(cert_file, 'w') as crt:
+                crt.write(ca_cert)
+            subprocess.check_call(['update-ca-certificates', '--fresh'])

charmhelpers/contrib/hahelpers/cluster.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 #
 # Copyright 2012 Canonical Ltd.
@@ -41,10 +39,11 @@ from charmhelpers.core.hookenv import (
     relation_get,
     config as config_get,
     INFO,
-    ERROR,
+    DEBUG,
     WARNING,
     unit_get,
-    is_leader as juju_is_leader
+    is_leader as juju_is_leader,
+    status_set,
 )
 from charmhelpers.core.decorators import (
     retry_on_exception,
@@ -60,6 +59,10 @@ class HAIncompleteConfig(Exception):
     pass
 
 
+class HAIncorrectConfig(Exception):
+    pass
+
+
 class CRMResourceNotFound(Exception):
     pass
 
@@ -274,27 +277,71 @@ def get_hacluster_config(exclude_keys=None):
     Obtains all relevant configuration from charm configuration required
     for initiating a relation to hacluster:
 
-        ha-bindiface, ha-mcastport, vip
+        ha-bindiface, ha-mcastport, vip, os-internal-hostname,
+        os-admin-hostname, os-public-hostname, os-access-hostname
 
     param: exclude_keys: list of setting key(s) to be excluded.
     returns: dict: A dict containing settings keyed by setting name.
-    raises: HAIncompleteConfig if settings are missing.
+    raises: HAIncompleteConfig if settings are missing or incorrect.
     '''
-    settings = ['ha-bindiface', 'ha-mcastport', 'vip']
+    settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname',
+                'os-admin-hostname', 'os-public-hostname', 'os-access-hostname']
     conf = {}
     for setting in settings:
         if exclude_keys and setting in exclude_keys:
             continue
 
         conf[setting] = config_get(setting)
-    missing = []
+
-    [missing.append(s) for s, v in six.iteritems(conf) if v is None]
+    if not valid_hacluster_config():
-    if missing:
+        raise HAIncorrectConfig('Insufficient or incorrect config data to '
-        log('Insufficient config data to configure hacluster.', level=ERROR)
+                                'configure hacluster.')
-        raise HAIncompleteConfig
     return conf
 
 
+def valid_hacluster_config():
+    '''
+    Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname
+    must be set.
+
+    Note: ha-bindiface and ha-macastport both have defaults and will always
+    be set. We only care that either vip or dns-ha is set.
+
+    :returns: boolean: valid config returns true.
+    raises: HAIncompatibileConfig if settings conflict.
+    raises: HAIncompleteConfig if settings are missing.
+    '''
+    vip = config_get('vip')
+    dns = config_get('dns-ha')
+    if not(bool(vip) ^ bool(dns)):
+        msg = ('HA: Either vip or dns-ha must be set but not both in order to '
+               'use high availability')
+        status_set('blocked', msg)
+        raise HAIncorrectConfig(msg)
+
+    # If dns-ha then one of os-*-hostname must be set
+    if dns:
+        dns_settings = ['os-internal-hostname', 'os-admin-hostname',
+                        'os-public-hostname', 'os-access-hostname']
+        # At this point it is unknown if one or all of the possible
+        # network spaces are in HA. Validate at least one is set which is
+        # the minimum required.
+        for setting in dns_settings:
+            if config_get(setting):
+                log('DNS HA: At least one hostname is set {}: {}'
+                    ''.format(setting, config_get(setting)),
+                    level=DEBUG)
+                return True
+
+        msg = ('DNS HA: At least one os-*-hostname(s) must be set to use '
+               'DNS HA')
+        status_set('blocked', msg)
+        raise HAIncompleteConfig(msg)
+
+    log('VIP HA: VIP is set {}'.format(vip), level=DEBUG)
+    return True
+
+
 def canonical_url(configs, vip_setting='vip'):
     '''
     Returns the correct HTTP URL to this host given the state of HTTPS

charmhelpers/contrib/network/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/network/ip.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import glob
 import re
@@ -22,25 +20,38 @@ import socket
 
 from functools import partial
 
-from charmhelpers.core.hookenv import unit_get
 from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
+    config,
     log,
+    network_get_primary_address,
+    unit_get,
     WARNING,
 )
 
+from charmhelpers.core.host import (
+    lsb_release,
+    CompareHostReleases,
+)
+
 try:
     import netifaces
 except ImportError:
     apt_update(fatal=True)
-    apt_install('python-netifaces', fatal=True)
+    if six.PY2:
+        apt_install('python-netifaces', fatal=True)
+    else:
+        apt_install('python3-netifaces', fatal=True)
     import netifaces
 
 try:
     import netaddr
 except ImportError:
     apt_update(fatal=True)
-    apt_install('python-netaddr', fatal=True)
+    if six.PY2:
+        apt_install('python-netaddr', fatal=True)
+    else:
+        apt_install('python3-netaddr', fatal=True)
     import netaddr
 
 
@@ -53,15 +64,33 @@ def _validate_cidr(network):
 
 
 def no_ip_found_error_out(network):
-    errmsg = ("No IP address found in network: %s" % network)
+    errmsg = ("No IP address found in network(s): %s" % network)
     raise ValueError(errmsg)
 
 
+def _get_ipv6_network_from_address(address):
+    """Get an netaddr.IPNetwork for the given IPv6 address
+    :param address: a dict as returned by netifaces.ifaddresses
+    :returns netaddr.IPNetwork: None if the address is a link local or loopback
+    address
+    """
+    if address['addr'].startswith('fe80') or address['addr'] == "::1":
+        return None
+
+    prefix = address['netmask'].split("/")
+    if len(prefix) > 1:
+        netmask = prefix[1]
+    else:
+        netmask = address['netmask']
+    return netaddr.IPNetwork("%s/%s" % (address['addr'],
+                                        netmask))
+
+
 def get_address_in_network(network, fallback=None, fatal=False):
     """Get an IPv4 or IPv6 address within the network from the host.
 
     :param network (str): CIDR presentation format. For example,
-        '192.168.1.0/24'.
+        '192.168.1.0/24'. Supports multiple networks as a space-delimited list.
     :param fallback (str): If no address is found, return fallback.
     :param fatal (boolean): If no address is found, fallback is not
         set and fatal is True then exit(1).
@@ -75,25 +104,25 @@ def get_address_in_network(network, fallback=None, fatal=False):
         else:
             return None
 
-    _validate_cidr(network)
+    networks = network.split() or [network]
-    network = netaddr.IPNetwork(network)
+    for network in networks:
-    for iface in netifaces.interfaces():
+        _validate_cidr(network)
-        addresses = netifaces.ifaddresses(iface)
+        network = netaddr.IPNetwork(network)
-        if network.version == 4 and netifaces.AF_INET in addresses:
+        for iface in netifaces.interfaces():
-            addr = addresses[netifaces.AF_INET][0]['addr']
+            addresses = netifaces.ifaddresses(iface)
-            netmask = addresses[netifaces.AF_INET][0]['netmask']
+            if network.version == 4 and netifaces.AF_INET in addresses:
-            cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+                for addr in addresses[netifaces.AF_INET]:
-            if cidr in network:
-                return str(cidr.ip)
-
-        if network.version == 6 and netifaces.AF_INET6 in addresses:
-            for addr in addresses[netifaces.AF_INET6]:
-                if not addr['addr'].startswith('fe80'):
                     cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
                                                         addr['netmask']))
                     if cidr in network:
                         return str(cidr.ip)
 
+            if network.version == 6 and netifaces.AF_INET6 in addresses:
+                for addr in addresses[netifaces.AF_INET6]:
+                    cidr = _get_ipv6_network_from_address(addr)
+                    if cidr and cidr in network:
+                        return str(cidr.ip)
+
     if fallback is not None:
         return fallback
 
@@ -168,18 +197,18 @@ def _get_for_address(address, key):
 
         if address.version == 6 and netifaces.AF_INET6 in addresses:
             for addr in addresses[netifaces.AF_INET6]:
-                if not addr['addr'].startswith('fe80'):
+                network = _get_ipv6_network_from_address(addr)
-                    network = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                if not network:
-                                                           addr['netmask']))
+                    continue
-                    cidr = network.cidr
-                    if address in cidr:
-                        if key == 'iface':
-                            return iface
-                        elif key == 'netmask' and cidr:
-                            return str(cidr).split('/')[1]
-                        else:
-                            return addr[key]
 
+                cidr = network.cidr
+                if address in cidr:
+                    if key == 'iface':
+                        return iface
+                    elif key == 'netmask' and cidr:
+                        return str(cidr).split('/')[1]
+                    else:
+                        return addr[key]
     return None
 
 
@@ -189,6 +218,15 @@ get_iface_for_address = partial(_get_for_address, key='iface')
 get_netmask_for_address = partial(_get_for_address, key='netmask')
 
 
+def resolve_network_cidr(ip_address):
+    '''
+    Resolves the full address cidr of an ip_address based on
+    configured network interfaces
+    '''
+    netmask = get_netmask_for_address(ip_address)
+    return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr)
+
+
 def format_ipv6_addr(address):
     """If address is IPv6, wrap it in '[]' otherwise return None.
 
@@ -203,7 +241,16 @@ def format_ipv6_addr(address):
 
 def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
                    fatal=True, exc_list=None):
-    """Return the assigned IP address for a given interface, if any."""
+    """Return the assigned IP address for a given interface, if any.
+
+    :param iface: network interface on which address(es) are expected to
+                  be found.
+    :param inet_type: inet address family
+    :param inc_aliases: include alias interfaces in search
+    :param fatal: if True, raise exception if address not found
+    :param exc_list: list of addresses to ignore
+    :return: list of ip addresses
+    """
     # Extract nic if passed /dev/ethX
     if '/' in iface:
         iface = iface.split('/')[-1]
@@ -304,6 +351,14 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
     We currently only support scope global IPv6 addresses i.e. non-temporary
     addresses. If no global IPv6 address is found, return the first one found
     in the ipv6 address list.
+
+    :param iface: network interface on which ipv6 address(es) are expected to
+                  be found.
+    :param inc_aliases: include alias interfaces in search
+    :param fatal: if True, raise exception if address not found
+    :param exc_list: list of addresses to ignore
+    :param dynamic_only: only recognise dynamic addresses
+    :return: list of ipv6 addresses
     """
     addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
                                inc_aliases=inc_aliases, fatal=fatal,
@@ -325,7 +380,7 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
             cmd = ['ip', 'addr', 'show', iface]
             out = subprocess.check_output(cmd).decode('UTF-8')
             if dynamic_only:
-                key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
+                key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*")
             else:
                 key = re.compile("inet6 (.+)/[0-9]+ scope global.*")
 
@@ -377,10 +432,10 @@ def is_ip(address):
     Returns True if address is a valid IP address.
     """
     try:
-        # Test to see if already an IPv4 address
+        # Test to see if already an IPv4/IPv6 address
-        socket.inet_aton(address)
+        address = netaddr.IPAddress(address)
         return True
-    except socket.error:
+    except (netaddr.AddrFormatError, ValueError):
         return False
 
 
@@ -388,7 +443,10 @@ def ns_query(address):
     try:
         import dns.resolver
     except ImportError:
-        apt_install('python-dnspython')
+        if six.PY2:
+            apt_install('python-dnspython', fatal=True)
+        else:
+            apt_install('python3-dnspython', fatal=True)
         import dns.resolver
 
     if isinstance(address, dns.name.Name):
@@ -398,7 +456,11 @@ def ns_query(address):
     else:
         return None
 
-    answers = dns.resolver.query(address, rtype)
+    try:
+        answers = dns.resolver.query(address, rtype)
+    except dns.resolver.NXDOMAIN:
+        return None
+
     if answers:
         return str(answers[0])
     return None
@@ -432,7 +494,10 @@ def get_hostname(address, fqdn=True):
         try:
             import dns.reversename
         except ImportError:
-            apt_install("python-dnspython")
+            if six.PY2:
+                apt_install("python-dnspython", fatal=True)
+            else:
+                apt_install("python3-dnspython", fatal=True)
             import dns.reversename
 
         rev = dns.reversename.from_address(address)
@@ -454,3 +519,56 @@ def get_hostname(address, fqdn=True):
             return result
     else:
         return result.split('.')[0]
+
+
+def port_has_listener(address, port):
+    """
+    Returns True if the address:port is open and being listened to,
+    else False.
+
+    @param address: an IP address or hostname
+    @param port: integer port
+
+    Note calls 'zc' via a subprocess shell
+    """
+    cmd = ['nc', '-z', address, str(port)]
+    result = subprocess.call(cmd)
+    return not(bool(result))
+
+
+def assert_charm_supports_ipv6():
+    """Check whether we are able to support charms ipv6."""
+    release = lsb_release()['DISTRIB_CODENAME'].lower()
+    if CompareHostReleases(release) < "trusty":
+        raise Exception("IPv6 is not supported in the charms for Ubuntu "
+                        "versions less than Trusty 14.04")
+
+
+def get_relation_ip(interface, config_override=None):
+    """Return this unit's IP for the given relation.
+
+    Allow for an arbitrary interface to use with network-get to select an IP.
+    Handle all address selection options including configuration parameter
+    override and IPv6.
+
+    Usage: get_relation_ip('amqp', config_override='access-network')
+
+    @param interface: string name of the relation.
+    @param config_override: string name of the config option for network
+           override. Supports legacy network override configuration parameters.
+    @raises Exception if prefer-ipv6 is configured but IPv6 unsupported.
+    @returns IPv6 or IPv4 address
+    """
+
+    fallback = get_host_ip(unit_get('private-address'))
+    if config('prefer-ipv6'):
+        assert_charm_supports_ipv6()
+        return get_ipv6_addr()[0]
+    elif config_override and config(config_override):
+        return get_address_in_network(config(config_override),
+                                      fallback)
+    else:
+        try:
+            return network_get_primary_address(interface)
+        except NotImplementedError:
+            return fallback

charmhelpers/contrib/openstack/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/openstack/alternatives.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 ''' Helper for managing alternatives for file conflict resolution '''
 

charmhelpers/contrib/openstack/amulet/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -1,25 +1,29 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,15 +32,31 @@ class OpenStackAmuletDeployment(AmuletDeployment):
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
-        # Note(coreycb): this needs to be changed when new next branches come
+
-        # out.
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
-        self.current_next = "trusty"
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
 
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
@@ -45,43 +65,82 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
 
-        # Charms outside the lp:~openstack-charmers namespace
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
-        base_charms = ['mysql', 'mongodb', 'nrpe']
 
-        # Force these charms to current series even when using an older series.
+        # Charms outside the ~openstack-charmers
-        # ie. Use trusty/nrpe even when series is precise, as the P charm
+        base_charms = {
-        # does not possess the necessary external master config and hooks.
+            'mysql': ['trusty'],
-        force_series_current = ['nrpe']
+            'mongodb': ['trusty'],
-
+            'nrpe': ['trusty', 'xenial'],
-        if self.series in ['precise', 'trusty']:
+        }
-            base_series = self.series
-        else:
-            base_series = self.current_next
 
         for svc in other_services:
-            if svc['name'] in force_series_current:
-                base_series = self.current_next
             # If a location has been explicitly set, use it
             if svc.get('location'):
                 continue
-            if self.stable:
+            if svc['name'] in base_charms:
-                temp = 'lp:charms/{}/{}'
+                # NOTE: not all charms have support for all series we
-                svc['location'] = temp.format(base_series,
+                #       want/need to test against, so fix to most recent
-                                              svc['name'])
+                #       that each base charm supports
+                target_series = self.series
+                if self.series not in base_charms[svc['name']]:
+                    target_series = base_charms[svc['name']][-1]
+                svc['location'] = 'cs:{}/{}'.format(target_series,
+                                                    svc['name'])
+            elif self.stable:
+                svc['location'] = 'cs:{}/{}'.format(self.series,
+                                                    svc['name'])
             else:
-                if svc['name'] in base_charms:
+                svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format(
-                    temp = 'lp:charms/{}/{}'
+                    self.series,
-                    svc['location'] = temp.format(base_series,
+                    svc['name']
-                                                  svc['name'])
+                )
-                else:
-                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
-                    svc['location'] = temp.format(self.current_next,
-                                                  svc['name'])
 
         return other_services
 
-    def _add_services(self, this_service, other_services):
+    def _add_services(self, this_service, other_services, use_source=None,
-        """Add services to the deployment and set openstack-origin/source."""
+                      no_origin=None):
+        """Add services to the deployment and optionally set
+        openstack-origin/source.
+
+        :param this_service dict: Service dictionary describing the service
+                                  whose amulet tests are being run
+        :param other_services dict: List of service dictionaries describing
+                                    the services needed to support the target
+                                    service
+        :param use_source list: List of services which use the 'source' config
+                                option rather than 'openstack-origin'
+        :param no_origin list: List of services which do not support setting
+                               the Cloud Archive.
+        Service Dict:
+            {
+                'name': str charm-name,
+                'units': int number of units,
+                'constraints': dict of juju constraints,
+                'location': str location of charm,
+            }
+        eg
+        this_service = {
+            'name': 'openvswitch-odl',
+            'constraints': {'mem': '8G'},
+        }
+        other_services = [
+            {
+                'name': 'nova-compute',
+                'units': 2,
+                'constraints': {'mem': '4G'},
+                'location': cs:~bob/xenial/nova-compute
+            },
+            {
+                'name': 'mysql',
+                'constraints': {'mem': '2G'},
+            },
+            {'neutron-api-odl'}]
+        use_source = ['mysql']
+        no_origin = ['neutron-api-odl']
+        """
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -90,12 +149,22 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         services = other_services
         services.append(this_service)
 
+        use_source = use_source or []
+        no_origin = no_origin or []
+
         # Charms which should use the source config option
-        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+        use_source = list(set(
-                      'ceph-osd', 'ceph-radosgw']
+            use_source + ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+                          'ceph-osd', 'ceph-radosgw', 'ceph-mon',
+                          'ceph-proxy', 'percona-cluster', 'lxd']))
 
         # Charms which can not use openstack-origin, ie. many subordinates
-        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
+        no_origin = list(set(
+            no_origin + ['cinder-ceph', 'hacluster', 'neutron-openvswitch',
+                         'nrpe', 'openvswitch-odl', 'neutron-api-odl',
+                         'odl-controller', 'cinder-backup', 'nexentaedge-data',
+                         'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw',
+                         'cinder-nexentaedge', 'nexentaedge-mgmt']))
 
         if self.openstack:
             for svc in services:
@@ -111,9 +180,79 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 
@@ -121,25 +260,21 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            release.
            """
         # Must be ordered by OpenStack release (not by Ubuntu release):
-        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+        (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
-         self.precise_havana, self.precise_icehouse,
+         self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
-         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
+         self.yakkety_newton, self.xenial_ocata, self.zesty_ocata) = range(9)
-         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
-         self.wily_liberty) = range(12)
 
         releases = {
-            ('precise', None): self.precise_essex,
-            ('precise', 'cloud:precise-folsom'): self.precise_folsom,
-            ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
-            ('precise', 'cloud:precise-havana'): self.precise_havana,
-            ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
             ('trusty', None): self.trusty_icehouse,
-            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
             ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
-            ('utopic', None): self.utopic_juno,
+            ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
-            ('vivid', None): self.vivid_kilo,
+            ('xenial', None): self.xenial_mitaka,
-            ('wily', None): self.wily_liberty}
+            ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
+            ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
+            ('yakkety', None): self.yakkety_newton,
+            ('zesty', None): self.zesty_ocata,
+        }
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -148,14 +283,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Return a string representing the openstack release.
            """
         releases = OrderedDict([
-            ('precise', 'essex'),
-            ('quantal', 'folsom'),
-            ('raring', 'grizzly'),
-            ('saucy', 'havana'),
             ('trusty', 'icehouse'),
-            ('utopic', 'juno'),
+            ('xenial', 'mitaka'),
-            ('vivid', 'kilo'),
+            ('yakkety', 'newton'),
-            ('wily', 'liberty'),
+            ('zesty', 'ocata'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

charmhelpers/contrib/openstack/amulet/utils.py

@@ -1,42 +1,52 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import amulet
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
+import urlparse
 
 import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
-import novaclient.v1_1.client as nova_client
+from keystoneclient.auth.identity import v3 as keystone_id_v3
+from keystoneclient import session as keystone_session
+from keystoneclient.v3 import client as keystone_client_v3
+from novaclient import exceptions
+
+import novaclient.client as nova_client
+import novaclient
 import pika
 import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
+from charmhelpers.core.decorators import retry_on_exception
+from charmhelpers.core.host import CompareHostReleases
 
 DEBUG = logging.DEBUG
 ERROR = logging.ERROR
 
+NOVA_CLIENT_VERSION = "2"
+
 
 class OpenStackAmuletUtils(AmuletUtils):
     """OpenStack amulet utilities.
@@ -78,6 +88,56 @@ class OpenStackAmuletUtils(AmuletUtils):
         if not found:
             return 'endpoint not found'
 
+    def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port,
+                                  public_port, expected):
+        """Validate keystone v3 endpoint data.
+
+        Validate the v3 endpoint data which has changed from v2.  The
+        ports are used to find the matching endpoint.
+
+        The new v3 endpoint data looks like:
+
+        [<Endpoint enabled=True,
+                   id=0432655fc2f74d1e9fa17bdaa6f6e60b,
+                   interface=admin,
+                   links={u'self': u'<RESTful URL of this endpoint>'},
+                   region=RegionOne,
+                   region_id=RegionOne,
+                   service_id=17f842a0dc084b928e476fafe67e4095,
+                   url=http://10.5.6.5:9312>,
+         <Endpoint enabled=True,
+                   id=6536cb6cb92f4f41bf22b079935c7707,
+                   interface=admin,
+                   links={u'self': u'<RESTful url of this endpoint>'},
+                   region=RegionOne,
+                   region_id=RegionOne,
+                   service_id=72fc8736fb41435e8b3584205bb2cfa3,
+                   url=http://10.5.6.6:35357/v3>,
+                   ... ]
+        """
+        self.log.debug('Validating v3 endpoint data...')
+        self.log.debug('actual: {}'.format(repr(endpoints)))
+        found = []
+        for ep in endpoints:
+            self.log.debug('endpoint: {}'.format(repr(ep)))
+            if ((admin_port in ep.url and ep.interface == 'admin') or
+                    (internal_port in ep.url and ep.interface == 'internal') or
+                    (public_port in ep.url and ep.interface == 'public')):
+                found.append(ep.interface)
+                # note we ignore the links member.
+                actual = {'id': ep.id,
+                          'region': ep.region,
+                          'region_id': ep.region_id,
+                          'interface': self.not_null,
+                          'url': ep.url,
+                          'service_id': ep.service_id, }
+                ret = self._validate_dict_data(expected, actual)
+                if ret:
+                    return 'unexpected endpoint data - {}'.format(ret)
+
+        if len(found) != 3:
+            return 'Unexpected number of endpoints found'
+
     def validate_svc_catalog_endpoint_data(self, expected, actual):
         """Validate service catalog endpoint data.
 
@@ -95,6 +155,72 @@ class OpenStackAmuletUtils(AmuletUtils):
                 return "endpoint {} does not exist".format(k)
         return ret
 
+    def validate_v3_svc_catalog_endpoint_data(self, expected, actual):
+        """Validate the keystone v3 catalog endpoint data.
+
+        Validate a list of dictinaries that make up the keystone v3 service
+        catalogue.
+
+        It is in the form of:
+
+
+        {u'identity': [{u'id': u'48346b01c6804b298cdd7349aadb732e',
+                        u'interface': u'admin',
+                        u'region': u'RegionOne',
+                        u'region_id': u'RegionOne',
+                        u'url': u'http://10.5.5.224:35357/v3'},
+                       {u'id': u'8414f7352a4b47a69fddd9dbd2aef5cf',
+                        u'interface': u'public',
+                        u'region': u'RegionOne',
+                        u'region_id': u'RegionOne',
+                        u'url': u'http://10.5.5.224:5000/v3'},
+                       {u'id': u'd5ca31440cc24ee1bf625e2996fb6a5b',
+                        u'interface': u'internal',
+                        u'region': u'RegionOne',
+                        u'region_id': u'RegionOne',
+                        u'url': u'http://10.5.5.224:5000/v3'}],
+         u'key-manager': [{u'id': u'68ebc17df0b045fcb8a8a433ebea9e62',
+                           u'interface': u'public',
+                           u'region': u'RegionOne',
+                           u'region_id': u'RegionOne',
+                           u'url': u'http://10.5.5.223:9311'},
+                          {u'id': u'9cdfe2a893c34afd8f504eb218cd2f9d',
+                           u'interface': u'internal',
+                           u'region': u'RegionOne',
+                           u'region_id': u'RegionOne',
+                           u'url': u'http://10.5.5.223:9311'},
+                          {u'id': u'f629388955bc407f8b11d8b7ca168086',
+                           u'interface': u'admin',
+                           u'region': u'RegionOne',
+                           u'region_id': u'RegionOne',
+                           u'url': u'http://10.5.5.223:9312'}]}
+
+        Note, that an added complication is that the order of admin, public,
+        internal against 'interface' in each region.
+
+        Thus, the function sorts the expected and actual lists using the
+        interface key as a sort key, prior to the comparison.
+        """
+        self.log.debug('Validating v3 service catalog endpoint data...')
+        self.log.debug('actual: {}'.format(repr(actual)))
+        for k, v in six.iteritems(expected):
+            if k in actual:
+                l_expected = sorted(v, key=lambda x: x['interface'])
+                l_actual = sorted(actual[k], key=lambda x: x['interface'])
+                if len(l_actual) != len(l_expected):
+                    return ("endpoint {} has differing number of interfaces "
+                            " - expected({}), actual({})"
+                            .format(k, len(l_expected), len(l_actual)))
+                for i_expected, i_actual in zip(l_expected, l_actual):
+                    self.log.debug("checking interface {}"
+                                   .format(i_expected['interface']))
+                    ret = self._validate_dict_data(i_expected, i_actual)
+                    if ret:
+                        return self.endpoint_error(k, ret)
+            else:
+                return "endpoint {} does not exist".format(k)
+        return ret
+
     def validate_tenant_data(self, expected, actual):
         """Validate tenant data.
 
@@ -138,7 +264,7 @@ class OpenStackAmuletUtils(AmuletUtils):
                 return "role {} does not exist".format(e['name'])
         return ret
 
-    def validate_user_data(self, expected, actual):
+    def validate_user_data(self, expected, actual, api_version=None):
         """Validate user data.
 
            Validate a list of actual user data vs a list of expected user
@@ -149,10 +275,15 @@ class OpenStackAmuletUtils(AmuletUtils):
         for e in expected:
             found = False
             for act in actual:
-                a = {'enabled': act.enabled, 'name': act.name,
+                if e['name'] == act.name:
-                     'email': act.email, 'tenantId': act.tenantId,
+                    a = {'enabled': act.enabled, 'name': act.name,
-                     'id': act.id}
+                         'email': act.email, 'id': act.id}
-                if e['name'] == a['name']:
+                    if api_version == 3:
+                        a['default_project_id'] = getattr(act,
+                                                          'default_project_id',
+                                                          'none')
+                    else:
+                        a['tenantId'] = act.tenantId
                     found = True
                     ret = self._validate_dict_data(e, a)
                     if ret:
@@ -176,34 +307,115 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    @retry_on_exception(5, base_delay=10)
+    def keystone_wait_for_propagation(self, sentry_relation_pairs,
+                                      api_version):
+        """Iterate over list of sentry and relation tuples and verify that
+           api_version has the expected value.
+
+        :param sentry_relation_pairs: list of sentry, relation name tuples used
+                                      for monitoring propagation of relation
+                                      data
+        :param api_version: api_version to expect in relation data
+        :returns: None if successful.  Raise on error.
+        """
+        for (sentry, relation_name) in sentry_relation_pairs:
+            rel = sentry.relation('identity-service',
+                                  relation_name)
+            self.log.debug('keystone relation data: {}'.format(rel))
+            if rel['api_version'] != str(api_version):
+                raise Exception("api_version not propagated through relation"
+                                " data yet ('{}' != '{}')."
+                                "".format(rel['api_version'], api_version))
+
+    def keystone_configure_api_version(self, sentry_relation_pairs, deployment,
+                                       api_version):
+        """Configure preferred-api-version of keystone in deployment and
+           monitor provided list of relation objects for propagation
+           before returning to caller.
+
+        :param sentry_relation_pairs: list of sentry, relation tuples used for
+                                      monitoring propagation of relation data
+        :param deployment: deployment to configure
+        :param api_version: value preferred-api-version will be set to
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug("Setting keystone preferred-api-version: '{}'"
+                       "".format(api_version))
+
+        config = {'preferred-api-version': api_version}
+        deployment.d.configure('keystone', config)
+        self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
+
     def authenticate_cinder_admin(self, keystone_sentry, username,
                                   password, tenant):
         """Authenticates admin user with cinder."""
         # NOTE(beisner): cinder python client doesn't accept tokens.
-        service_ip = \
+        keystone_ip = keystone_sentry.info['public-address']
-            keystone_sentry.relation('shared-db',
+        ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
-                                     'mysql:shared-db')['private-address']
-        ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
         return cinder_client.Client(username, password, tenant, ept)
 
+    def authenticate_keystone(self, keystone_ip, username, password,
+                              api_version=False, admin_port=False,
+                              user_domain_name=None, domain_name=None,
+                              project_domain_name=None, project_name=None):
+        """Authenticate with Keystone"""
+        self.log.debug('Authenticating with keystone...')
+        port = 5000
+        if admin_port:
+            port = 35357
+        base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
+                                        port)
+        if not api_version or api_version == 2:
+            ep = base_ep + "/v2.0"
+            return keystone_client.Client(username=username, password=password,
+                                          tenant_name=project_name,
+                                          auth_url=ep)
+        else:
+            ep = base_ep + "/v3"
+            auth = keystone_id_v3.Password(
+                user_domain_name=user_domain_name,
+                username=username,
+                password=password,
+                domain_name=domain_name,
+                project_domain_name=project_domain_name,
+                project_name=project_name,
+                auth_url=ep
+            )
+            return keystone_client_v3.Client(
+                session=keystone_session.Session(auth=auth)
+            )
+
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
-                                    tenant):
+                                    tenant=None, api_version=None,
+                                    keystone_ip=None):
         """Authenticates admin user with the keystone admin endpoint."""
         self.log.debug('Authenticating keystone admin...')
-        unit = keystone_sentry
+        if not keystone_ip:
-        service_ip = unit.relation('shared-db',
+            keystone_ip = keystone_sentry.info['public-address']
-                                   'mysql:shared-db')['private-address']
+
-        ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
+        user_domain_name = None
-        return keystone_client.Client(username=user, password=password,
+        domain_name = None
-                                      tenant_name=tenant, auth_url=ep)
+        if api_version == 3:
+            user_domain_name = 'admin_domain'
+            domain_name = user_domain_name
+
+        return self.authenticate_keystone(keystone_ip, user, password,
+                                          project_name=tenant,
+                                          api_version=api_version,
+                                          user_domain_name=user_domain_name,
+                                          domain_name=domain_name,
+                                          admin_port=True)
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
         self.log.debug('Authenticating keystone user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return keystone_client.Client(username=user, password=password,
+        keystone_ip = urlparse.urlparse(ep).hostname
-                                      tenant_name=tenant, auth_url=ep)
+
+        return self.authenticate_keystone(keystone_ip, user, password,
+                                          project_name=tenant)
 
     def authenticate_glance_admin(self, keystone):
         """Authenticates admin user with glance."""
@@ -224,8 +436,14 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return nova_client.Client(username=user, api_key=password,
+        if novaclient.__version__[0] >= "7":
-                                  project_id=tenant, auth_url=ep)
+            return nova_client.Client(NOVA_CLIENT_VERSION,
+                                      username=user, password=password,
+                                      project_name=tenant, auth_url=ep)
+        else:
+            return nova_client.Client(NOVA_CLIENT_VERSION,
+                                      username=user, api_key=password,
+                                      project_id=tenant, auth_url=ep)
 
     def authenticate_swift_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with swift api."""
@@ -238,6 +456,16 @@ class OpenStackAmuletUtils(AmuletUtils):
                                       tenant_name=tenant,
                                       auth_version='2.0')
 
+    def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
+                      ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
+        """Create the specified flavor."""
+        try:
+            nova.flavors.find(name=name)
+        except (exceptions.NotFound, exceptions.NoUniqueMatch):
+            self.log.debug('Creating flavor ({})'.format(name))
+            nova.flavors.create(name, ram, vcpus, disk, flavorid,
+                                ephemeral, swap, rxtx_factor, is_public)
+
     def create_cirros_image(self, glance, image_name):
         """Download the latest cirros image and upload it to glance,
         validate and return a resource pointer.
@@ -604,7 +832,22 @@ class OpenStackAmuletUtils(AmuletUtils):
                            '{}'.format(sample_type, samples))
             return None
 
-# rabbitmq/amqp specific helpers:
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
     def add_rmq_test_user(self, sentry_units,
                           username="testuser1", password="changeme"):
         """Add a test user via the first rmq juju unit, check connection as
@@ -805,7 +1048,10 @@ class OpenStackAmuletUtils(AmuletUtils):
         if port:
             config['ssl_port'] = port
 
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0
@@ -832,7 +1078,10 @@ class OpenStackAmuletUtils(AmuletUtils):
 
         # Disable RMQ SSL
         config = {'ssl': 'off'}
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0
@@ -881,7 +1130,8 @@ class OpenStackAmuletUtils(AmuletUtils):
                                                    retry_delay=5,
                                                    socket_timeout=1)
             connection = pika.BlockingConnection(parameters)
-            assert connection.server_properties['product'] == 'RabbitMQ'
+            assert connection.is_open is True
+            assert connection.is_closing is False
             self.log.debug('Connect OK')
             return connection
         except Exception as e:
@@ -961,3 +1211,70 @@ class OpenStackAmuletUtils(AmuletUtils):
         else:
             msg = 'No message retrieved.'
             amulet.raise_status(amulet.FAIL, msg)
+
+    def validate_memcache(self, sentry_unit, conf, os_release,
+                          earliest_release=5, section='keystone_authtoken',
+                          check_kvs=None):
+        """Check Memcache is running and is configured to be used
+
+        Example call from Amulet test:
+
+            def test_110_memcache(self):
+                u.validate_memcache(self.neutron_api_sentry,
+                                    '/etc/neutron/neutron.conf',
+                                    self._get_openstack_release())
+
+        :param sentry_unit: sentry unit
+        :param conf: OpenStack config file to check memcache settings
+        :param os_release: Current OpenStack release int code
+        :param earliest_release: Earliest Openstack release to check int code
+        :param section: OpenStack config file section to check
+        :param check_kvs: Dict of settings to check in config file
+        :returns: None
+        """
+        if os_release < earliest_release:
+            self.log.debug('Skipping memcache checks for deployment. {} <'
+                           'mitaka'.format(os_release))
+            return
+        _kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'}
+        self.log.debug('Checking memcached is running')
+        ret = self.validate_services_by_name({sentry_unit: ['memcached']})
+        if ret:
+            amulet.raise_status(amulet.FAIL, msg='Memcache running check'
+                                'failed {}'.format(ret))
+        else:
+            self.log.debug('OK')
+        self.log.debug('Checking memcache url is configured in {}'.format(
+            conf))
+        if self.validate_config_data(sentry_unit, conf, section, _kvs):
+            message = "Memcache config error in: {}".format(conf)
+            amulet.raise_status(amulet.FAIL, msg=message)
+        else:
+            self.log.debug('OK')
+        self.log.debug('Checking memcache configuration in '
+                       '/etc/memcached.conf')
+        contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
+                                           fatal=True)
+        ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
+        if CompareHostReleases(ubuntu_release) <= 'trusty':
+            memcache_listen_addr = 'ip6-localhost'
+        else:
+            memcache_listen_addr = '::1'
+        expected = {
+            '-p': '11211',
+            '-l': memcache_listen_addr}
+        found = []
+        for key, value in expected.items():
+            for line in contents.split('\n'):
+                if line.startswith(key):
+                    self.log.debug('Checking {} is set to {}'.format(
+                        key,
+                        value))
+                    assert value == line.split()[-1]
+                    self.log.debug(line.split()[-1])
+                    found.append(key)
+        if sorted(found) == sorted(expected.keys()):
+            self.log.debug('OK')
+        else:
+            message = "Memcache config error in: /etc/memcached.conf"
+            amulet.raise_status(amulet.FAIL, msg=message)

charmhelpers/contrib/openstack/context.py

@@ -1,29 +1,27 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import glob
 import json
+import math
 import os
 import re
 import time
 from base64 import b64decode
-from subprocess import check_call
+from subprocess import check_call, CalledProcessError
 
 import six
-import yaml
 
 from charmhelpers.fetch import (
     apt_install,
@@ -45,10 +43,12 @@ from charmhelpers.core.hookenv import (
     INFO,
     WARNING,
     ERROR,
+    status_set,
 )
 
 from charmhelpers.core.sysctl import create as sysctl_create
 from charmhelpers.core.strutils import bool_from_string
+from charmhelpers.contrib.openstack.exceptions import OSContextError
 
 from charmhelpers.core.host import (
     get_bond_master,
@@ -57,6 +57,10 @@ from charmhelpers.core.host import (
     get_nic_hwaddr,
     mkdir,
     write_file,
+    pwgen,
+    lsb_release,
+    CompareHostReleases,
+    is_container,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     determine_apache_port,
@@ -86,15 +90,28 @@ from charmhelpers.contrib.network.ip import (
     is_address_in_network,
     is_bridge_member,
 )
-from charmhelpers.contrib.openstack.utils import get_host_ip
+from charmhelpers.contrib.openstack.utils import (
+    config_flags_parser,
+    get_host_ip,
+    git_determine_usr_bin,
+    git_determine_python_path,
+    enable_memcache,
+)
+from charmhelpers.core.unitdata import kv
+
+try:
+    import psutil
+except ImportError:
+    if six.PY2:
+        apt_install('python-psutil', fatal=True)
+    else:
+        apt_install('python3-psutil', fatal=True)
+    import psutil
+
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 ADDRESS_TYPES = ['admin', 'internal', 'public']
 
 
-class OSContextError(Exception):
-    pass
-
-
 def ensure_packages(packages):
     """Install but do not upgrade required plugin packages."""
     required = filter_installed_packages(packages)
@@ -115,83 +132,6 @@ def context_complete(ctxt):
     return True
 
 
-def config_flags_parser(config_flags):
-    """Parses config flags string into dict.
-
-    This parsing method supports a few different formats for the config
-    flag values to be parsed:
-
-      1. A string in the simple format of key=value pairs, with the possibility
-         of specifying multiple key value pairs within the same string. For
-         example, a string in the format of 'key1=value1, key2=value2' will
-         return a dict of:
-
-             {'key1': 'value1',
-              'key2': 'value2'}.
-
-      2. A string in the above format, but supporting a comma-delimited list
-         of values for the same key. For example, a string in the format of
-         'key1=value1, key2=value3,value4,value5' will return a dict of:
-
-             {'key1', 'value1',
-              'key2', 'value2,value3,value4'}
-
-      3. A string containing a colon character (:) prior to an equal
-         character (=) will be treated as yaml and parsed as such. This can be
-         used to specify more complex key value pairs. For example,
-         a string in the format of 'key1: subkey1=value1, subkey2=value2' will
-         return a dict of:
-
-             {'key1', 'subkey1=value1, subkey2=value2'}
-
-    The provided config_flags string may be a list of comma-separated values
-    which themselves may be comma-separated list of values.
-    """
-    # If we find a colon before an equals sign then treat it as yaml.
-    # Note: limit it to finding the colon first since this indicates assignment
-    # for inline yaml.
-    colon = config_flags.find(':')
-    equals = config_flags.find('=')
-    if colon > 0:
-        if colon < equals or equals < 0:
-            return yaml.safe_load(config_flags)
-
-    if config_flags.find('==') >= 0:
-        log("config_flags is not in expected format (key=value)", level=ERROR)
-        raise OSContextError
-
-    # strip the following from each value.
-    post_strippers = ' ,'
-    # we strip any leading/trailing '=' or ' ' from the string then
-    # split on '='.
-    split = config_flags.strip(' =').split('=')
-    limit = len(split)
-    flags = {}
-    for i in range(0, limit - 1):
-        current = split[i]
-        next = split[i + 1]
-        vindex = next.rfind(',')
-        if (i == limit - 2) or (vindex < 0):
-            value = next
-        else:
-            value = next[:vindex]
-
-        if i == 0:
-            key = current
-        else:
-            # if this not the first entry, expect an embedded key.
-            index = current.rfind(',')
-            if index < 0:
-                log("Invalid config value(s) at index %s" % (i), level=ERROR)
-                raise OSContextError
-            key = current[index + 1:]
-
-        # Add to collection.
-        flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
-
-    return flags
-
-
 class OSContextGenerator(object):
     """Base class for all context generators."""
     interfaces = []
@@ -217,7 +157,8 @@ class OSContextGenerator(object):
 
         if self.missing_data:
             self.complete = False
-            log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
+            log('Missing required data: %s' % ' '.join(self.missing_data),
+                level=INFO)
         else:
             self.complete = True
         return self.complete
@@ -275,8 +216,9 @@ class SharedDBContext(OSContextGenerator):
                 hostname_key = "{}_hostname".format(self.relation_prefix)
             else:
                 hostname_key = "hostname"
-            access_hostname = get_address_in_network(access_network,
+            access_hostname = get_address_in_network(
-                                                     unit_get('private-address'))
+                access_network,
+                unit_get('private-address'))
             set_hostname = relation_get(attribute=hostname_key,
                                         unit=local_unit())
             if set_hostname != access_hostname:
@@ -370,7 +312,10 @@ def db_ssl(rdata, ctxt, ssl_dir):
 
 class IdentityServiceContext(OSContextGenerator):
 
-    def __init__(self, service=None, service_user=None, rel_name='identity-service'):
+    def __init__(self,
+                 service=None,
+                 service_user=None,
+                 rel_name='identity-service'):
         self.service = service
         self.service_user = service_user
         self.rel_name = rel_name
@@ -401,6 +346,7 @@ class IdentityServiceContext(OSContextGenerator):
                 auth_host = format_ipv6_addr(auth_host) or auth_host
                 svc_protocol = rdata.get('service_protocol') or 'http'
                 auth_protocol = rdata.get('auth_protocol') or 'http'
+                api_version = rdata.get('api_version') or '2.0'
                 ctxt.update({'service_port': rdata.get('service_port'),
                              'service_host': serv_host,
                              'auth_host': auth_host,
@@ -409,7 +355,12 @@ class IdentityServiceContext(OSContextGenerator):
                              'admin_user': rdata.get('service_username'),
                              'admin_password': rdata.get('service_password'),
                              'service_protocol': svc_protocol,
-                             'auth_protocol': auth_protocol})
+                             'auth_protocol': auth_protocol,
+                             'api_version': api_version})
+
+                if float(api_version) > 2:
+                    ctxt.update({'admin_domain_name':
+                                 rdata.get('service_domain')})
 
                 if self.context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
@@ -451,16 +402,20 @@ class AMQPContext(OSContextGenerator):
         for rid in relation_ids(self.rel_name):
             ha_vip_only = False
             self.related = True
+            transport_hosts = None
+            rabbitmq_port = '5672'
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
                     vip = relation_get('vip', rid=rid, unit=unit)
                     vip = format_ipv6_addr(vip) or vip
                     ctxt['rabbitmq_host'] = vip
+                    transport_hosts = [vip]
                 else:
                     host = relation_get('private-address', rid=rid, unit=unit)
                     host = format_ipv6_addr(host) or host
                     ctxt['rabbitmq_host'] = host
+                    transport_hosts = [host]
 
                 ctxt.update({
                     'rabbitmq_user': username,
@@ -472,6 +427,7 @@ class AMQPContext(OSContextGenerator):
                 ssl_port = relation_get('ssl_port', rid=rid, unit=unit)
                 if ssl_port:
                     ctxt['rabbit_ssl_port'] = ssl_port
+                    rabbitmq_port = ssl_port
 
                 ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit)
                 if ssl_ca:
@@ -508,7 +464,19 @@ class AMQPContext(OSContextGenerator):
                     host = format_ipv6_addr(host) or host
                     rabbitmq_hosts.append(host)
 
-                ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
+                rabbitmq_hosts = sorted(rabbitmq_hosts)
+                ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
+                transport_hosts = rabbitmq_hosts
+
+            if transport_hosts:
+                transport_url_hosts = ','.join([
+                    "{}:{}@{}:{}".format(ctxt['rabbitmq_user'],
+                                         ctxt['rabbitmq_password'],
+                                         host_,
+                                         rabbitmq_port)
+                    for host_ in transport_hosts])
+                ctxt['transport_url'] = "rabbit://{}/{}".format(
+                    transport_url_hosts, vhost)
 
         oslo_messaging_flags = conf.get('oslo-messaging-flags', None)
         if oslo_messaging_flags:
@@ -540,13 +508,16 @@ class CephContext(OSContextGenerator):
                     ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
                 if not ctxt.get('key'):
                     ctxt['key'] = relation_get('key', rid=rid, unit=unit)
-                ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
+
+                ceph_addrs = relation_get('ceph-public-address', rid=rid,
+                                          unit=unit)
+                if ceph_addrs:
+                    for addr in ceph_addrs.split(' '):
+                        mon_hosts.append(format_ipv6_addr(addr) or addr)
+                else:
+                    priv_addr = relation_get('private-address', rid=rid,
                                              unit=unit)
-                unit_priv_addr = relation_get('private-address', rid=rid,
+                    mon_hosts.append(format_ipv6_addr(priv_addr) or priv_addr)
-                                              unit=unit)
-                ceph_addr = ceph_pub_addr or unit_priv_addr
-                ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
-                mon_hosts.append(ceph_addr)
 
         ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
 
@@ -626,15 +597,28 @@ class HAProxyContext(OSContextGenerator):
         if config('haproxy-client-timeout'):
             ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
 
+        if config('haproxy-queue-timeout'):
+            ctxt['haproxy_queue_timeout'] = config('haproxy-queue-timeout')
+
+        if config('haproxy-connect-timeout'):
+            ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout')
+
         if config('prefer-ipv6'):
             ctxt['ipv6'] = True
             ctxt['local_host'] = 'ip6-localhost'
             ctxt['haproxy_host'] = '::'
-            ctxt['stat_port'] = ':::8888'
         else:
             ctxt['local_host'] = '127.0.0.1'
             ctxt['haproxy_host'] = '0.0.0.0'
-            ctxt['stat_port'] = ':8888'
+
+        ctxt['stat_port'] = '8888'
+
+        db = kv()
+        ctxt['stat_password'] = db.get('stat-password')
+        if not ctxt['stat_password']:
+            ctxt['stat_password'] = db.set('stat-password',
+                                           pwgen(32))
+            db.flush()
 
         for frontend in cluster_hosts:
             if (len(cluster_hosts[frontend]['backends']) > 1 or
@@ -698,7 +682,7 @@ class ApacheSSLContext(OSContextGenerator):
     service_namespace = None
 
     def enable_modules(self):
-        cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http']
+        cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http', 'headers']
         check_call(cmd)
 
     def configure_cert(self, cn=None):
@@ -952,6 +936,19 @@ class NeutronContext(OSContextGenerator):
                     'config': config}
         return ovs_ctxt
 
+    def midonet_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        midonet_config = neutron_plugin_attribute(self.plugin, 'config',
+                                                  self.network_manager)
+        mido_ctxt = {'core_plugin': driver,
+                     'neutron_plugin': 'midonet',
+                     'neutron_security_groups': self.neutron_security_groups,
+                     'local_ip': unit_private_ip(),
+                     'config': midonet_config}
+
+        return mido_ctxt
+
     def __call__(self):
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
@@ -973,6 +970,8 @@ class NeutronContext(OSContextGenerator):
             ctxt.update(self.nuage_ctxt())
         elif self.plugin == 'plumgrid':
             ctxt.update(self.pg_ctxt())
+        elif self.plugin == 'midonet':
+            ctxt.update(self.midonet_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:
@@ -1073,6 +1072,20 @@ class OSConfigFlagContext(OSContextGenerator):
                 config_flags_parser(config_flags)}
 
 
+class LibvirtConfigFlagsContext(OSContextGenerator):
+    """
+    This context provides support for extending
+    the libvirt section through user-defined flags.
+    """
+    def __call__(self):
+        ctxt = {}
+        libvirt_flags = config('libvirt-flags')
+        if libvirt_flags:
+            ctxt['libvirt_flags'] = config_flags_parser(
+                libvirt_flags)
+        return ctxt
+
+
 class SubordinateConfigContext(OSContextGenerator):
 
     """
@@ -1105,7 +1118,7 @@ class SubordinateConfigContext(OSContextGenerator):
 
         ctxt = {
             ... other context ...
-            'subordinate_config': {
+            'subordinate_configuration': {
                 'DEFAULT': {
                     'key1': 'value1',
                 },
@@ -1146,22 +1159,23 @@ class SubordinateConfigContext(OSContextGenerator):
                     try:
                         sub_config = json.loads(sub_config)
                     except:
-                        log('Could not parse JSON from subordinate_config '
+                        log('Could not parse JSON from '
-                            'setting from %s' % rid, level=ERROR)
+                            'subordinate_configuration setting from %s'
+                            % rid, level=ERROR)
                         continue
 
                     for service in self.services:
                         if service not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
+                            log('Found subordinate_configuration on %s but it '
-                                'nothing for %s service' % (rid, service),
+                                'contained nothing for %s service'
-                                level=INFO)
+                                % (rid, service), level=INFO)
                             continue
 
                         sub_config = sub_config[service]
                         if self.config_file not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
+                            log('Found subordinate_configuration on %s but it '
-                                'nothing for %s' % (rid, self.config_file),
+                                'contained nothing for %s'
-                                level=INFO)
+                                % (rid, self.config_file), level=INFO)
                             continue
 
                         sub_config = sub_config[self.config_file]
@@ -1208,21 +1222,72 @@ class BindHostContext(OSContextGenerator):
             return {'bind_host': '0.0.0.0'}
 
 
+MAX_DEFAULT_WORKERS = 4
+DEFAULT_MULTIPLIER = 2
+
+
 class WorkerConfigContext(OSContextGenerator):
 
     @property
     def num_cpus(self):
-        try:
+        # NOTE: use cpu_count if present (16.04 support)
-            from psutil import NUM_CPUS
+        if hasattr(psutil, 'cpu_count'):
-        except ImportError:
+            return psutil.cpu_count()
-            apt_install('python-psutil', fatal=True)
+        else:
-            from psutil import NUM_CPUS
+            return psutil.NUM_CPUS
-
-        return NUM_CPUS
 
     def __call__(self):
-        multiplier = config('worker-multiplier') or 0
+        multiplier = config('worker-multiplier') or DEFAULT_MULTIPLIER
-        ctxt = {"workers": self.num_cpus * multiplier}
+        count = int(self.num_cpus * multiplier)
+        if multiplier > 0 and count == 0:
+            count = 1
+
+        if config('worker-multiplier') is None and is_container():
+            # NOTE(jamespage): Limit unconfigured worker-multiplier
+            #                  to MAX_DEFAULT_WORKERS to avoid insane
+            #                  worker configuration in LXD containers
+            #                  on large servers
+            # Reference: https://pad.lv/1665270
+            count = min(count, MAX_DEFAULT_WORKERS)
+
+        ctxt = {"workers": count}
+        return ctxt
+
+
+class WSGIWorkerConfigContext(WorkerConfigContext):
+
+    def __init__(self, name=None, script=None, admin_script=None,
+                 public_script=None, process_weight=1.00,
+                 admin_process_weight=0.75, public_process_weight=0.25):
+        self.service_name = name
+        self.user = name
+        self.group = name
+        self.script = script
+        self.admin_script = admin_script
+        self.public_script = public_script
+        self.process_weight = process_weight
+        self.admin_process_weight = admin_process_weight
+        self.public_process_weight = public_process_weight
+
+    def __call__(self):
+        multiplier = config('worker-multiplier') or 1
+        total_processes = self.num_cpus * multiplier
+        ctxt = {
+            "service_name": self.service_name,
+            "user": self.user,
+            "group": self.group,
+            "script": self.script,
+            "admin_script": self.admin_script,
+            "public_script": self.public_script,
+            "processes": int(math.ceil(self.process_weight * total_processes)),
+            "admin_processes": int(math.ceil(self.admin_process_weight *
+                                             total_processes)),
+            "public_processes": int(math.ceil(self.public_process_weight *
+                                              total_processes)),
+            "threads": 1,
+            "usr_bin": git_determine_usr_bin(),
+            "python_path": git_determine_python_path(),
+        }
         return ctxt
 
 
@@ -1364,7 +1429,7 @@ class DataPortContext(NeutronPortContext):
             normalized.update({port: port for port in resolved
                                if port in ports})
             if resolved:
-                return {bridge: normalized[port] for port, bridge in
+                return {normalized[port]: bridge for port, bridge in
                         six.iteritems(portmap) if port in normalized.keys()}
 
         return None
@@ -1375,8 +1440,8 @@ class PhyNICMTUContext(DataPortContext):
     def __call__(self):
         ctxt = {}
         mappings = super(PhyNICMTUContext, self).__call__()
-        if mappings and mappings.values():
+        if mappings and mappings.keys():
-            ports = mappings.values()
+            ports = sorted(mappings.keys())
             napi_settings = NeutronAPIContext()()
             mtu = napi_settings.get('network_device_mtu')
             all_ports = set()
@@ -1421,7 +1486,147 @@ class NetworkServiceContext(OSContextGenerator):
                     rdata.get('service_protocol') or 'http',
                     'auth_protocol':
                     rdata.get('auth_protocol') or 'http',
+                    'api_version':
+                    rdata.get('api_version') or '2.0',
                 }
                 if self.context_complete(ctxt):
                     return ctxt
         return {}
+
+
+class InternalEndpointContext(OSContextGenerator):
+    """Internal endpoint context.
+
+    This context provides the endpoint type used for communication between
+    services e.g. between Nova and Cinder internally. Openstack uses Public
+    endpoints by default so this allows admins to optionally use internal
+    endpoints.
+    """
+    def __call__(self):
+        return {'use_internal_endpoints': config('use-internal-endpoints')}
+
+
+class AppArmorContext(OSContextGenerator):
+    """Base class for apparmor contexts."""
+
+    def __init__(self, profile_name=None):
+        self._ctxt = None
+        self.aa_profile = profile_name
+        self.aa_utils_packages = ['apparmor-utils']
+
+    @property
+    def ctxt(self):
+        if self._ctxt is not None:
+            return self._ctxt
+        self._ctxt = self._determine_ctxt()
+        return self._ctxt
+
+    def _determine_ctxt(self):
+        """
+        Validate aa-profile-mode settings is disable, enforce, or complain.
+
+        :return ctxt: Dictionary of the apparmor profile or None
+        """
+        if config('aa-profile-mode') in ['disable', 'enforce', 'complain']:
+            ctxt = {'aa_profile_mode': config('aa-profile-mode'),
+                    'ubuntu_release': lsb_release()['DISTRIB_RELEASE']}
+            if self.aa_profile:
+                ctxt['aa_profile'] = self.aa_profile
+        else:
+            ctxt = None
+        return ctxt
+
+    def __call__(self):
+        return self.ctxt
+
+    def install_aa_utils(self):
+        """
+        Install packages required for apparmor configuration.
+        """
+        log("Installing apparmor utils.")
+        ensure_packages(self.aa_utils_packages)
+
+    def manually_disable_aa_profile(self):
+        """
+        Manually disable an apparmor profile.
+
+        If aa-profile-mode is set to disabled (default) this is required as the
+        template has been written but apparmor is yet unaware of the profile
+        and aa-disable aa-profile fails. Without this the profile would kick
+        into enforce mode on the next service restart.
+
+        """
+        profile_path = '/etc/apparmor.d'
+        disable_path = '/etc/apparmor.d/disable'
+        if not os.path.lexists(os.path.join(disable_path, self.aa_profile)):
+            os.symlink(os.path.join(profile_path, self.aa_profile),
+                       os.path.join(disable_path, self.aa_profile))
+
+    def setup_aa_profile(self):
+        """
+        Setup an apparmor profile.
+        The ctxt dictionary will contain the apparmor profile mode and
+        the apparmor profile name.
+        Makes calls out to aa-disable, aa-complain, or aa-enforce to setup
+        the apparmor profile.
+        """
+        self()
+        if not self.ctxt:
+            log("Not enabling apparmor Profile")
+            return
+        self.install_aa_utils()
+        cmd = ['aa-{}'.format(self.ctxt['aa_profile_mode'])]
+        cmd.append(self.ctxt['aa_profile'])
+        log("Setting up the apparmor profile for {} in {} mode."
+            "".format(self.ctxt['aa_profile'], self.ctxt['aa_profile_mode']))
+        try:
+            check_call(cmd)
+        except CalledProcessError as e:
+            # If aa-profile-mode is set to disabled (default) manual
+            # disabling is required as the template has been written but
+            # apparmor is yet unaware of the profile and aa-disable aa-profile
+            # fails. If aa-disable learns to read profile files first this can
+            # be removed.
+            if self.ctxt['aa_profile_mode'] == 'disable':
+                log("Manually disabling the apparmor profile for {}."
+                    "".format(self.ctxt['aa_profile']))
+                self.manually_disable_aa_profile()
+                return
+            status_set('blocked', "Apparmor profile {} failed to be set to {}."
+                                  "".format(self.ctxt['aa_profile'],
+                                            self.ctxt['aa_profile_mode']))
+            raise e
+
+
+class MemcacheContext(OSContextGenerator):
+    """Memcache context
+
+    This context provides options for configuring a local memcache client and
+    server
+    """
+
+    def __init__(self, package=None):
+        """
+        @param package: Package to examine to extrapolate OpenStack release.
+                        Used when charms have no openstack-origin config
+                        option (ie subordinates)
+        """
+        self.package = package
+
+    def __call__(self):
+        ctxt = {}
+        ctxt['use_memcache'] = enable_memcache(package=self.package)
+        if ctxt['use_memcache']:
+            # Trusty version of memcached does not support ::1 as a listen
+            # address so use host file entry instead
+            release = lsb_release()['DISTRIB_CODENAME'].lower()
+            if CompareHostReleases(release) > 'trusty':
+                ctxt['memcache_server'] = '::1'
+            else:
+                ctxt['memcache_server'] = 'ip6-localhost'
+            ctxt['memcache_server_formatted'] = '[::1]'
+            ctxt['memcache_port'] = '11211'
+            ctxt['memcache_url'] = 'inet6:{}:{}'.format(
+                ctxt['memcache_server_formatted'],
+                ctxt['memcache_port'])
+        return ctxt

charmhelpers/contrib/openstack/exceptions.py

@@ -0,0 +1,21 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class OSContextError(Exception):
+    """Raised when an error occurs during context generation.
+
+    This exception is principally used in contrib.openstack.context
+    """
+    pass

charmhelpers/contrib/openstack/files/__init__.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 # dummy __init__.py to fool syncer into thinking this is a syncable python
 # module

charmhelpers/contrib/openstack/files/check_haproxy.sh

@@ -9,15 +9,17 @@
 CRITICAL=0
 NOTACTIVE=''
 LOGFILE=/var/log/nagios/check_haproxy.log
-AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+AUTH=$(grep -r "stats auth" /etc/haproxy | awk 'NR=1{print $4}')
 
-for appserver in $(grep '    server' /etc/haproxy/haproxy.cfg | awk '{print $2'});
+typeset -i N_INSTANCES=0
+for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)
 do
-    output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK')
+    N_INSTANCES=N_INSTANCES+1
+    output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' --regex=",${appserver},.*,UP.*" -e ' 200 OK')
     if [ $? != 0 ]; then
         date >> $LOGFILE
         echo $output >> $LOGFILE
-        /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1
+        /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v | grep ",${appserver}," >> $LOGFILE 2>&1
         CRITICAL=1
         NOTACTIVE="${NOTACTIVE} $appserver"
     fi
@@ -28,5 +30,5 @@ if [ $CRITICAL = 1 ]; then
     exit 2
 fi
 
-echo "OK: All haproxy instances looking good"
+echo "OK: All haproxy instances ($N_INSTANCES) looking good"
 exit 0

charmhelpers/contrib/openstack/ha/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

charmhelpers/contrib/openstack/ha/utils.py

@@ -0,0 +1,139 @@
+# Copyright 2014-2016 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Copyright 2016 Canonical Ltd.
+#
+# Authors:
+#  Openstack Charmers <
+#
+
+"""
+Helpers for high availability.
+"""
+
+import re
+
+from charmhelpers.core.hookenv import (
+    log,
+    relation_set,
+    charm_name,
+    config,
+    status_set,
+    DEBUG,
+)
+
+from charmhelpers.core.host import (
+    lsb_release
+)
+
+from charmhelpers.contrib.openstack.ip import (
+    resolve_address,
+)
+
+
+class DNSHAException(Exception):
+    """Raised when an error occurs setting up DNS HA
+    """
+
+    pass
+
+
+def update_dns_ha_resource_params(resources, resource_params,
+                                  relation_id=None,
+                                  crm_ocf='ocf:maas:dns'):
+    """ Check for os-*-hostname settings and update resource dictionaries for
+    the HA relation.
+
+    @param resources: Pointer to dictionary of resources.
+                      Usually instantiated in ha_joined().
+    @param resource_params: Pointer to dictionary of resource parameters.
+                            Usually instantiated in ha_joined()
+    @param relation_id: Relation ID of the ha relation
+    @param crm_ocf: Corosync Open Cluster Framework resource agent to use for
+                    DNS HA
+    """
+
+    # Validate the charm environment for DNS HA
+    assert_charm_supports_dns_ha()
+
+    settings = ['os-admin-hostname', 'os-internal-hostname',
+                'os-public-hostname', 'os-access-hostname']
+
+    # Check which DNS settings are set and update dictionaries
+    hostname_group = []
+    for setting in settings:
+        hostname = config(setting)
+        if hostname is None:
+            log('DNS HA: Hostname setting {} is None. Ignoring.'
+                ''.format(setting),
+                DEBUG)
+            continue
+        m = re.search('os-(.+?)-hostname', setting)
+        if m:
+            networkspace = m.group(1)
+        else:
+            msg = ('Unexpected DNS hostname setting: {}. '
+                   'Cannot determine network space name'
+                   ''.format(setting))
+            status_set('blocked', msg)
+            raise DNSHAException(msg)
+
+        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace)
+        if hostname_key in hostname_group:
+            log('DNS HA: Resource {}: {} already exists in '
+                'hostname group - skipping'.format(hostname_key, hostname),
+                DEBUG)
+            continue
+
+        hostname_group.append(hostname_key)
+        resources[hostname_key] = crm_ocf
+        resource_params[hostname_key] = (
+            'params fqdn="{}" ip_address="{}" '
+            ''.format(hostname, resolve_address(endpoint_type=networkspace,
+                                                override=False)))
+
+    if len(hostname_group) >= 1:
+        log('DNS HA: Hostname group is set with {} as members. '
+            'Informing the ha relation'.format(' '.join(hostname_group)),
+            DEBUG)
+        relation_set(relation_id=relation_id, groups={
+            'grp_{}_hostnames'.format(charm_name()): ' '.join(hostname_group)})
+    else:
+        msg = 'DNS HA: Hostname group has no members.'
+        status_set('blocked', msg)
+        raise DNSHAException(msg)
+
+
+def assert_charm_supports_dns_ha():
+    """Validate prerequisites for DNS HA
+    The MAAS client is only available on Xenial or greater
+    """
+    if lsb_release().get('DISTRIB_RELEASE') < '16.04':
+        msg = ('DNS HA is only supported on 16.04 and greater '
+               'versions of Ubuntu.')
+        status_set('blocked', msg)
+        raise DNSHAException(msg)
+    return True
+
+
+def expect_ha():
+    """ Determine if the unit expects to be in HA
+
+    Check for VIP or dns-ha settings which indicate the unit should expect to
+    be related to hacluster.
+
+    @returns boolean
+    """
+    return config('vip') or config('dns-ha')

charmhelpers/contrib/openstack/ip.py

@@ -1,52 +1,62 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 from charmhelpers.core.hookenv import (
     config,
     unit_get,
     service_name,
+    network_get_primary_address,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
     is_address_in_network,
     is_ipv6,
     get_ipv6_addr,
+    resolve_network_cidr,
 )
 from charmhelpers.contrib.hahelpers.cluster import is_clustered
 
 PUBLIC = 'public'
 INTERNAL = 'int'
 ADMIN = 'admin'
+ACCESS = 'access'
 
 ADDRESS_MAP = {
     PUBLIC: {
+        'binding': 'public',
         'config': 'os-public-network',
         'fallback': 'public-address',
         'override': 'os-public-hostname',
     },
     INTERNAL: {
+        'binding': 'internal',
         'config': 'os-internal-network',
         'fallback': 'private-address',
         'override': 'os-internal-hostname',
     },
     ADMIN: {
+        'binding': 'admin',
         'config': 'os-admin-network',
         'fallback': 'private-address',
         'override': 'os-admin-hostname',
-    }
+    },
+    ACCESS: {
+        'binding': 'access',
+        'config': 'access-network',
+        'fallback': 'private-address',
+        'override': 'os-access-hostname',
+    },
 }
 
 
@@ -103,20 +113,23 @@ def _get_address_override(endpoint_type=PUBLIC):
         return addr_override.format(service_name=service_name())
 
 
-def resolve_address(endpoint_type=PUBLIC):
+def resolve_address(endpoint_type=PUBLIC, override=True):
     """Return unit address depending on net config.
 
     If unit is clustered with vip(s) and has net splits defined, return vip on
     correct network. If clustered with no nets defined, return primary vip.
 
     If not clustered, return unit address ensuring address is on configured net
-    split if one is configured.
+    split if one is configured, or a Juju 2.0 extra-binding has been used.
 
     :param endpoint_type: Network endpoing type
+    :param override: Accept hostname overrides or not
     """
-    resolved_address = _get_address_override(endpoint_type)
+    resolved_address = None
-    if resolved_address:
+    if override:
-        return resolved_address
+        resolved_address = _get_address_override(endpoint_type)
+        if resolved_address:
+            return resolved_address
 
     vips = config('vip')
     if vips:
@@ -125,23 +138,45 @@ def resolve_address(endpoint_type=PUBLIC):
     net_type = ADDRESS_MAP[endpoint_type]['config']
     net_addr = config(net_type)
     net_fallback = ADDRESS_MAP[endpoint_type]['fallback']
+    binding = ADDRESS_MAP[endpoint_type]['binding']
     clustered = is_clustered()
-    if clustered:
+
-        if not net_addr:
+    if clustered and vips:
-            # If no net-splits defined, we expect a single vip
+        if net_addr:
-            resolved_address = vips[0]
-        else:
             for vip in vips:
                 if is_address_in_network(net_addr, vip):
                     resolved_address = vip
                     break
+        else:
+            # NOTE: endeavour to check vips against network space
+            #       bindings
+            try:
+                bound_cidr = resolve_network_cidr(
+                    network_get_primary_address(binding)
+                )
+                for vip in vips:
+                    if is_address_in_network(bound_cidr, vip):
+                        resolved_address = vip
+                        break
+            except NotImplementedError:
+                # If no net-splits configured and no support for extra
+                # bindings/network spaces so we expect a single vip
+                resolved_address = vips[0]
     else:
         if config('prefer-ipv6'):
             fallback_addr = get_ipv6_addr(exc_list=vips)[0]
         else:
             fallback_addr = unit_get(net_fallback)
 
-        resolved_address = get_address_in_network(net_addr, fallback_addr)
+        if net_addr:
+            resolved_address = get_address_in_network(net_addr, fallback_addr)
+        else:
+            # NOTE: only try to use extra bindings if legacy network
+            #       configuration is not in use
+            try:
+                resolved_address = network_get_primary_address(binding)
+            except NotImplementedError:
+                resolved_address = fallback_addr
 
     if resolved_address is None:
         raise ValueError("Unable to resolve a suitable IP address based on "

charmhelpers/contrib/openstack/keystone.py

@@ -0,0 +1,178 @@
+#!/usr/bin/python
+#
+# Copyright 2017 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import six
+from charmhelpers.fetch import apt_install
+from charmhelpers.contrib.openstack.context import IdentityServiceContext
+from charmhelpers.core.hookenv import (
+    log,
+    ERROR,
+)
+
+
+def get_api_suffix(api_version):
+    """Return the formatted api suffix for the given version
+    @param api_version: version of the keystone endpoint
+    @returns the api suffix formatted according to the given api
+    version
+    """
+    return 'v2.0' if api_version in (2, "2.0") else 'v3'
+
+
+def format_endpoint(schema, addr, port, api_version):
+    """Return a formatted keystone endpoint
+    @param schema: http or https
+    @param addr: ipv4/ipv6 host of the keystone service
+    @param port: port of the keystone service
+    @param api_version: 2 or 3
+    @returns a fully formatted keystone endpoint
+    """
+    return '{}://{}:{}/{}/'.format(schema, addr, port,
+                                   get_api_suffix(api_version))
+
+
+def get_keystone_manager(endpoint, api_version, **kwargs):
+    """Return a keystonemanager for the correct API version
+
+    @param endpoint: the keystone endpoint to point client at
+    @param api_version: version of the keystone api the client should use
+    @param kwargs: token or username/tenant/password information
+    @returns keystonemanager class used for interrogating keystone
+    """
+    if api_version == 2:
+        return KeystoneManager2(endpoint, **kwargs)
+    if api_version == 3:
+        return KeystoneManager3(endpoint, **kwargs)
+    raise ValueError('No manager found for api version {}'.format(api_version))
+
+
+def get_keystone_manager_from_identity_service_context():
+    """Return a keystonmanager generated from a
+    instance of charmhelpers.contrib.openstack.context.IdentityServiceContext
+    @returns keystonamenager instance
+   """
+    context = IdentityServiceContext()()
+    if not context:
+        msg = "Identity service context cannot be generated"
+        log(msg, level=ERROR)
+        raise ValueError(msg)
+
+    endpoint = format_endpoint(context['service_protocol'],
+                               context['service_host'],
+                               context['service_port'],
+                               context['api_version'])
+
+    if context['api_version'] in (2, "2.0"):
+        api_version = 2
+    else:
+        api_version = 3
+
+    return get_keystone_manager(endpoint, api_version,
+                                username=context['admin_user'],
+                                password=context['admin_password'],
+                                tenant_name=context['admin_tenant_name'])
+
+
+class KeystoneManager(object):
+
+    def resolve_service_id(self, service_name=None, service_type=None):
+        """Find the service_id of a given service"""
+        services = [s._info for s in self.api.services.list()]
+
+        service_name = service_name.lower()
+        for s in services:
+            name = s['name'].lower()
+            if service_type and service_name:
+                if (service_name == name and service_type == s['type']):
+                    return s['id']
+            elif service_name and service_name == name:
+                return s['id']
+            elif service_type and service_type == s['type']:
+                return s['id']
+        return None
+
+    def service_exists(self, service_name=None, service_type=None):
+        """Determine if the given service exists on the service list"""
+        return self.resolve_service_id(service_name, service_type) is not None
+
+
+class KeystoneManager2(KeystoneManager):
+
+    def __init__(self, endpoint, **kwargs):
+        try:
+            from keystoneclient.v2_0 import client
+            from keystoneclient.auth.identity import v2
+            from keystoneclient import session
+        except ImportError:
+            if six.PY2:
+                apt_install(["python-keystoneclient"], fatal=True)
+            else:
+                apt_install(["python3-keystoneclient"], fatal=True)
+
+            from keystoneclient.v2_0 import client
+            from keystoneclient.auth.identity import v2
+            from keystoneclient import session
+
+        self.api_version = 2
+
+        token = kwargs.get("token", None)
+        if token:
+            api = client.Client(endpoint=endpoint, token=token)
+        else:
+            auth = v2.Password(username=kwargs.get("username"),
+                               password=kwargs.get("password"),
+                               tenant_name=kwargs.get("tenant_name"),
+                               auth_url=endpoint)
+            sess = session.Session(auth=auth)
+            api = client.Client(session=sess)
+
+        self.api = api
+
+
+class KeystoneManager3(KeystoneManager):
+
+    def __init__(self, endpoint, **kwargs):
+        try:
+            from keystoneclient.v3 import client
+            from keystoneclient.auth import token_endpoint
+            from keystoneclient import session
+            from keystoneclient.auth.identity import v3
+        except ImportError:
+            if six.PY2:
+                apt_install(["python-keystoneclient"], fatal=True)
+            else:
+                apt_install(["python3-keystoneclient"], fatal=True)
+
+            from keystoneclient.v3 import client
+            from keystoneclient.auth import token_endpoint
+            from keystoneclient import session
+            from keystoneclient.auth.identity import v3
+
+        self.api_version = 3
+
+        token = kwargs.get("token", None)
+        if token:
+            auth = token_endpoint.Token(endpoint=endpoint,
+                                        token=token)
+            sess = session.Session(auth=auth)
+        else:
+            auth = v3.Password(auth_url=endpoint,
+                               user_id=kwargs.get("username"),
+                               password=kwargs.get("password"),
+                               project_id=kwargs.get("tenant_name"))
+            sess = session.Session(auth=auth)
+
+        self.api = client.Client(session=sess)

charmhelpers/contrib/openstack/neutron.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 # Various utilies for dealing with Neutron and the renaming from Quantum.
 
@@ -25,7 +23,10 @@ from charmhelpers.core.hookenv import (
     ERROR,
 )
 
-from charmhelpers.contrib.openstack.utils import os_release
+from charmhelpers.contrib.openstack.utils import (
+    os_release,
+    CompareOpenStackReleases,
+)
 
 
 def headers_package():
@@ -34,6 +35,7 @@ def headers_package():
     kver = check_output(['uname', '-r']).decode('UTF-8').strip()
     return 'linux-headers-%s' % kver
 
+
 QUANTUM_CONF_DIR = '/etc/quantum'
 
 
@@ -50,7 +52,7 @@ def determine_dkms_package():
     if kernel_version() >= (3, 13):
         return []
     else:
-        return ['openvswitch-datapath-dkms']
+        return [headers_package(), 'openvswitch-datapath-dkms']
 
 
 # legacy
@@ -70,7 +72,7 @@ def quantum_plugins():
                                         relation_prefix='neutron',
                                         ssl_dir=QUANTUM_CONF_DIR)],
             'services': ['quantum-plugin-openvswitch-agent'],
-            'packages': [[headers_package()] + determine_dkms_package(),
+            'packages': [determine_dkms_package(),
                          ['quantum-plugin-openvswitch-agent']],
             'server_packages': ['quantum-server',
                                 'quantum-plugin-openvswitch'],
@@ -93,6 +95,7 @@ def quantum_plugins():
         }
     }
 
+
 NEUTRON_CONF_DIR = '/etc/neutron'
 
 
@@ -111,7 +114,7 @@ def neutron_plugins():
                                         relation_prefix='neutron',
                                         ssl_dir=NEUTRON_CONF_DIR)],
             'services': ['neutron-plugin-openvswitch-agent'],
-            'packages': [[headers_package()] + determine_dkms_package(),
+            'packages': [determine_dkms_package(),
                          ['neutron-plugin-openvswitch-agent']],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-openvswitch'],
@@ -155,7 +158,7 @@ def neutron_plugins():
                                         relation_prefix='neutron',
                                         ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
-            'packages': [[headers_package()] + determine_dkms_package(),
+            'packages': [determine_dkms_package(),
                          ['neutron-plugin-cisco']],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-cisco'],
@@ -174,7 +177,7 @@ def neutron_plugins():
                          'neutron-dhcp-agent',
                          'nova-api-metadata',
                          'etcd'],
-            'packages': [[headers_package()] + determine_dkms_package(),
+            'packages': [determine_dkms_package(),
                          ['calico-compute',
                           'bird',
                           'neutron-dhcp-agent',
@@ -198,20 +201,35 @@ def neutron_plugins():
         },
         'plumgrid': {
             'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
-            'driver': 'neutron.plugins.plumgrid.plumgrid_plugin.plumgrid_plugin.NeutronPluginPLUMgridV2',
+            'driver': ('neutron.plugins.plumgrid.plumgrid_plugin'
+                       '.plumgrid_plugin.NeutronPluginPLUMgridV2'),
             'contexts': [
                 context.SharedDBContext(user=config('database-user'),
                                         database=config('database'),
                                         ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
-            'packages': [['plumgrid-lxc'],
+            'packages': ['plumgrid-lxc',
-                         ['iovisor-dkms']],
+                         'iovisor-dkms'],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-plumgrid'],
             'server_services': ['neutron-server']
+        },
+        'midonet': {
+            'config': '/etc/neutron/plugins/midonet/midonet.ini',
+            'driver': 'midonet.neutron.plugin.MidonetPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [determine_dkms_package()],
+            'server_packages': ['neutron-server',
+                                'python-neutron-plugin-midonet'],
+            'server_services': ['neutron-server']
         }
     }
-    if release >= 'icehouse':
+    if CompareOpenStackReleases(release) >= 'icehouse':
         # NOTE: patch in ml2 plugin for icehouse onwards
         plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
         plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
@@ -219,6 +237,27 @@ def neutron_plugins():
                                              'neutron-plugin-ml2']
         # NOTE: patch in vmware renames nvp->nsx for icehouse onwards
         plugins['nvp'] = plugins['nsx']
+    if CompareOpenStackReleases(release) >= 'kilo':
+        plugins['midonet']['driver'] = (
+            'neutron.plugins.midonet.plugin.MidonetPluginV2')
+    if CompareOpenStackReleases(release) >= 'liberty':
+        plugins['midonet']['driver'] = (
+            'midonet.neutron.plugin_v1.MidonetPluginV2')
+        plugins['midonet']['server_packages'].remove(
+            'python-neutron-plugin-midonet')
+        plugins['midonet']['server_packages'].append(
+            'python-networking-midonet')
+        plugins['plumgrid']['driver'] = (
+            'networking_plumgrid.neutron.plugins'
+            '.plugin.NeutronPluginPLUMgridV2')
+        plugins['plumgrid']['server_packages'].remove(
+            'neutron-plugin-plumgrid')
+    if CompareOpenStackReleases(release) >= 'mitaka':
+        plugins['nsx']['server_packages'].remove('neutron-plugin-vmware')
+        plugins['nsx']['server_packages'].append('python-vmware-nsx')
+        plugins['nsx']['config'] = '/etc/neutron/nsx.ini'
+        plugins['vsp']['driver'] = (
+            'nuage_neutron.plugins.nuage.plugin.NuagePlugin')
     return plugins
 
 
@@ -310,10 +349,10 @@ def parse_bridge_mappings(mappings):
 def parse_data_port_mappings(mappings, default_bridge='br-data'):
     """Parse data port mappings.
 
-    Mappings must be a space-delimited list of port:bridge mappings.
+    Mappings must be a space-delimited list of bridge:port.
 
-    Returns dict of the form {port:bridge} where port may be an mac address or
+    Returns dict of the form {port:bridge} where ports may be mac addresses or
-    interface name.
+    interface names.
     """
 
     # NOTE(dosaboy): we use rvalue for key to allow multiple values to be

charmhelpers/contrib/openstack/templates/__init__.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 # dummy __init__.py to fool syncer into thinking this is a syncable python
 # module

charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -5,6 +5,8 @@ global
     user haproxy
     group haproxy
     spread-checks 0
+    stats socket /var/run/haproxy/admin.sock mode 600 level admin
+    stats timeout 2m
 
 defaults
     log global
@@ -12,27 +14,35 @@ defaults
     option tcplog
     option dontlognull
     retries 3
-    timeout queue 1000
+{%- if haproxy_queue_timeout %}
-    timeout connect 1000
+    timeout queue {{ haproxy_queue_timeout }}
-{% if haproxy_client_timeout -%}
+{%- else %}
+    timeout queue 5000
+{%- endif %}
+{%- if haproxy_connect_timeout %}
+    timeout connect {{ haproxy_connect_timeout }}
+{%- else %}
+    timeout connect 5000
+{%- endif %}
+{%- if haproxy_client_timeout %}
     timeout client {{ haproxy_client_timeout }}
-{% else -%}
+{%- else %}
     timeout client 30000
-{% endif -%}
+{%- endif %}
-
+{%- if haproxy_server_timeout %}
-{% if haproxy_server_timeout -%}
     timeout server {{ haproxy_server_timeout }}
-{% else -%}
+{%- else %}
     timeout server 30000
-{% endif -%}
+{%- endif %}
 
-listen stats {{ stat_port }}
+listen stats
+    bind {{ local_host }}:{{ stat_port }}
     mode http
     stats enable
     stats hide-version
     stats realm Haproxy\ Statistics
     stats uri /
-    stats auth admin:password
+    stats auth admin:{{ stat_password }}
 
 {% if frontends -%}
 {% for service, ports in service_ports.items() -%}
@@ -50,6 +60,15 @@ frontend tcp-in_{{ service }}
 {% for frontend in frontends -%}
 backend {{ service }}_{{ frontend }}
     balance leastconn
+    {% if backend_options -%}
+    {% if backend_options[service] -%}
+    {% for option in backend_options[service] -%}
+    {% for key, value in option.items() -%}
+    {{ key }} {{ value }}
+    {% endfor -%}
+    {% endfor -%}
+    {% endif -%}
+    {% endif -%}
     {% for unit, address in frontends[frontend]['backends'].items() -%}
     server {{ unit }} {{ address }}:{{ ports[1] }} check
     {% endfor %}

charmhelpers/contrib/openstack/templates/memcached.conf

@@ -0,0 +1,53 @@
+###############################################################################
+# [ WARNING ]
+# memcached configuration file maintained by Juju
+# local changes may be overwritten.
+###############################################################################
+
+# memcached default config file
+# 2003 - Jay Bonci <jaybonci@debian.org>
+# This configuration file is read by the start-memcached script provided as
+# part of the Debian GNU/Linux distribution.
+
+# Run memcached as a daemon. This command is implied, and is not needed for the
+# daemon to run. See the README.Debian that comes with this package for more
+# information.
+-d
+
+# Log memcached's output to /var/log/memcached
+logfile /var/log/memcached.log
+
+# Be verbose
+# -v
+
+# Be even more verbose (print client commands as well)
+# -vv
+
+# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
+# Note that the daemon will grow to this size, but does not start out holding this much
+# memory
+-m 64
+
+# Default connection port is 11211
+-p {{ memcache_port }}
+
+# Run the daemon as root. The start-memcached will default to running as root if no
+# -u command is present in this config file
+-u memcache
+
+# Specify which IP address to listen on. The default is to listen on all IP addresses
+# This parameter is one of the only security measures that memcached has, so make sure
+# it's listening on a firewalled interface.
+-l {{ memcache_server }}
+
+# Limit the number of simultaneous incoming connections. The daemon default is 1024
+# -c 1024
+
+# Lock down all paged memory. Consult with the README and homepage before you do this
+# -k
+
+# Return error when memory is exhausted (rather than removing items)
+# -M
+
+# Maximize core file limit
+# -r

charmhelpers/contrib/openstack/templates/openstack_https_frontend

@@ -6,11 +6,16 @@ Listen {{ ext_port }}
 <VirtualHost {{ address }}:{{ ext }}>
     ServerName {{ endpoint }}
     SSLEngine on
+    SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2
+    SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM
     SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
+    # See LP 1484489 - this is to support <= 2.4.7 and >= 2.4.8
+    SSLCertificateChainFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
     SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
     ProxyPass / http://localhost:{{ int }}/
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
+    RequestHeader set X-Forwarded-Proto "https"
 </VirtualHost>
 {% endfor -%}
 <Proxy *>

charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf

@@ -6,11 +6,16 @@ Listen {{ ext_port }}
 <VirtualHost {{ address }}:{{ ext }}>
     ServerName {{ endpoint }}
     SSLEngine on
+    SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2
+    SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM
     SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
+    # See LP 1484489 - this is to support <= 2.4.7 and >= 2.4.8
+    SSLCertificateChainFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
     SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
     ProxyPass / http://localhost:{{ int }}/
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
+    RequestHeader set X-Forwarded-Proto "https"
 </VirtualHost>
 {% endfor -%}
 <Proxy *>

charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -1,9 +1,12 @@
 {% if auth_host -%}
 [keystone_authtoken]
-identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
-auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
-admin_tenant_name = {{ admin_tenant_name }}
+auth_plugin = password
-admin_user = {{ admin_user }}
+project_domain_id = default
-admin_password = {{ admin_password }}
+user_domain_id = default
+project_name = {{ admin_tenant_name }}
+username = {{ admin_user }}
+password = {{ admin_password }}
 signing_dir = {{ signing_dir }}
 {% endif -%}

charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy

@@ -0,0 +1,10 @@
+{% if auth_host -%}
+[keystone_authtoken]
+# Juno specific config (Bug #1557223)
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
+identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
+admin_tenant_name = {{ admin_tenant_name }}
+admin_user = {{ admin_user }}
+admin_password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+{% endif -%}

charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -0,0 +1,20 @@
+{% if auth_host -%}
+[keystone_authtoken]
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
+auth_type = password
+{% if api_version == "3" -%}
+project_domain_name = {{ admin_domain_name }}
+user_domain_name = {{ admin_domain_name }}
+{% else -%}
+project_domain_name = default
+user_domain_name = default
+{% endif -%}
+project_name = {{ admin_tenant_name }}
+username = {{ admin_user }}
+password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+{% if use_memcache == true %}
+memcached_servers = {{ memcache_url }}
+{% endif -%}
+{% endif -%}

charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -0,0 +1,100 @@
+# Configuration file maintained by Juju. Local changes may be overwritten.
+
+{% if port -%}
+Listen {{ port }}
+{% endif -%}
+
+{% if admin_port -%}
+Listen {{ admin_port }}
+{% endif -%}
+
+{% if public_port -%}
+Listen {{ public_port }}
+{% endif -%}
+
+{% if port -%}
+<VirtualHost *:{{ port }}>
+    WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}
+    WSGIScriptAlias / {{ script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}
+
+{% if admin_port -%}
+<VirtualHost *:{{ admin_port }}>
+    WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}-admin
+    WSGIScriptAlias / {{ admin_script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}
+
+{% if public_port -%}
+<VirtualHost *:{{ public_port }}>
+    WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}-public
+    WSGIScriptAlias / {{ public_script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}

charmhelpers/contrib/openstack/templating.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 
@@ -30,7 +28,10 @@ try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
     apt_update(fatal=True)
-    apt_install('python-jinja2', fatal=True)
+    if six.PY2:
+        apt_install('python-jinja2', fatal=True)
+    else:
+        apt_install('python3-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
 
@@ -209,7 +210,10 @@ class OSConfigRenderer(object):
             # if this code is running, the object is created pre-install hook.
             # jinja2 shouldn't get touched until the module is reloaded on next
             # hook execution, with proper jinja2 bits successfully imported.
-            apt_install('python-jinja2')
+            if six.PY2:
+                apt_install('python-jinja2')
+            else:
+                apt_install('python3-jinja2')
 
     def register(self, config_file, contexts):
         """

charmhelpers/contrib/python/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/python/packages.py

@@ -3,36 +3,53 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import six
 import subprocess
+import sys
 
 from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import charm_dir, log
 
-try:
-    from pip import main as pip_execute
-except ImportError:
-    apt_update()
-    apt_install('python-pip')
-    from pip import main as pip_execute
-
 __author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
 
 
+def pip_execute(*args, **kwargs):
+    """Overriden pip_execute() to stop sys.path being changed.
+
+    The act of importing main from the pip module seems to cause add wheels
+    from the /usr/share/python-wheels which are installed by various tools.
+    This function ensures that sys.path remains the same after the call is
+    executed.
+    """
+    try:
+        _path = sys.path
+        try:
+            from pip import main as _pip_execute
+        except ImportError:
+            apt_update()
+            if six.PY2:
+                apt_install('python-pip')
+            else:
+                apt_install('python3-pip')
+            from pip import main as _pip_execute
+        _pip_execute(*args, **kwargs)
+    finally:
+        sys.path = _path
+
+
 def parse_options(given, available):
     """Given a set of options, check if available"""
     for key, value in sorted(given.items()):
@@ -42,8 +59,12 @@ def parse_options(given, available):
             yield "--{0}={1}".format(key, value)
 
 
-def pip_install_requirements(requirements, **options):
+def pip_install_requirements(requirements, constraints=None, **options):
-    """Install a requirements file """
+    """Install a requirements file.
+
+    :param constraints: Path to pip constraints file.
+    http://pip.readthedocs.org/en/stable/user_guide/#constraints-files
+    """
     command = ["install"]
 
     available_options = ('proxy', 'src', 'log', )
@@ -51,12 +72,18 @@ def pip_install_requirements(requirements, **options):
         command.append(option)
 
     command.append("-r {0}".format(requirements))
-    log("Installing from file: {} with options: {}".format(requirements,
+    if constraints:
-                                                           command))
+        command.append("-c {0}".format(constraints))
+        log("Installing from file: {} with constraints {} "
+            "and options: {}".format(requirements, constraints, command))
+    else:
+        log("Installing from file: {} with options: {}".format(requirements,
+                                                               command))
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=None,
+                constraints=None, **options):
     """Install a python package"""
     if venv:
         venv_python = os.path.join(venv, 'bin/pip')
@@ -71,6 +98,9 @@ def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
     if upgrade:
         command.append('--upgrade')
 
+    if constraints:
+        command.extend(['-c', constraints])
+
     if isinstance(package, list):
         command.extend(package)
     else:
@@ -110,7 +140,10 @@ def pip_list():
 
 def pip_create_virtualenv(path=None):
     """Create an isolated Python environment."""
-    apt_install('python-virtualenv')
+    if six.PY2:
+        apt_install('python-virtualenv')
+    else:
+        apt_install('python3-virtualenv')
 
     if path:
         venv_path = path

charmhelpers/contrib/storage/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/storage/linux/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/contrib/storage/linux/ceph.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 #
 # Copyright 2012 Canonical Ltd.
@@ -24,6 +22,11 @@
 #  Adam Gandelman <adamg@ubuntu.com>
 #
 
+import errno
+import hashlib
+import math
+import six
+
 import os
 import shutil
 import json
@@ -36,6 +39,8 @@ from subprocess import (
     CalledProcessError,
 )
 from charmhelpers.core.hookenv import (
+    config,
+    service_name,
     local_unit,
     relation_get,
     relation_ids,
@@ -60,6 +65,7 @@ from charmhelpers.fetch import (
 )
 
 from charmhelpers.core.kernel import modprobe
+from charmhelpers.contrib.openstack.utils import config_flags_parser
 
 KEYRING = '/etc/ceph/ceph.client.{}.keyring'
 KEYFILE = '/etc/ceph/ceph.client.{}.key'
@@ -73,6 +79,646 @@ err to syslog = {use_syslog}
 clog to syslog = {use_syslog}
 """
 
+# The number of placement groups per OSD to target for placement group
+# calculations. This number is chosen as 100 due to the ceph PG Calc
+# documentation recommending to choose 100 for clusters which are not
+# expected to increase in the foreseeable future. Since the majority of the
+# calculations are done on deployment, target the case of non-expanding
+# clusters as the default.
+DEFAULT_PGS_PER_OSD_TARGET = 100
+DEFAULT_POOL_WEIGHT = 10.0
+LEGACY_PG_COUNT = 200
+DEFAULT_MINIMUM_PGS = 2
+
+
+def validator(value, valid_type, valid_range=None):
+    """
+    Used to validate these: http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values
+    Example input:
+        validator(value=1,
+                  valid_type=int,
+                  valid_range=[0, 2])
+    This says I'm testing value=1.  It must be an int inclusive in [0,2]
+
+    :param value: The value to validate
+    :param valid_type: The type that value should be.
+    :param valid_range: A range of values that value can assume.
+    :return:
+    """
+    assert isinstance(value, valid_type), "{} is not a {}".format(
+        value,
+        valid_type)
+    if valid_range is not None:
+        assert isinstance(valid_range, list), \
+            "valid_range must be a list, was given {}".format(valid_range)
+        # If we're dealing with strings
+        if valid_type is six.string_types:
+            assert value in valid_range, \
+                "{} is not in the list {}".format(value, valid_range)
+        # Integer, float should have a min and max
+        else:
+            if len(valid_range) != 2:
+                raise ValueError(
+                    "Invalid valid_range list of {} for {}.  "
+                    "List must be [min,max]".format(valid_range, value))
+            assert value >= valid_range[0], \
+                "{} is less than minimum allowed value of {}".format(
+                    value, valid_range[0])
+            assert value <= valid_range[1], \
+                "{} is greater than maximum allowed value of {}".format(
+                    value, valid_range[1])
+
+
+class PoolCreationError(Exception):
+    """
+    A custom error to inform the caller that a pool creation failed.  Provides an error message
+    """
+
+    def __init__(self, message):
+        super(PoolCreationError, self).__init__(message)
+
+
+class Pool(object):
+    """
+    An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool.
+    Do not call create() on this base class as it will not do anything.  Instantiate a child class and call create().
+    """
+
+    def __init__(self, service, name):
+        self.service = service
+        self.name = name
+
+    # Create the pool if it doesn't exist already
+    # To be implemented by subclasses
+    def create(self):
+        pass
+
+    def add_cache_tier(self, cache_pool, mode):
+        """
+        Adds a new cache tier to an existing pool.
+        :param cache_pool: six.string_types.  The cache tier pool name to add.
+        :param mode: six.string_types. The caching mode to use for this pool.  valid range = ["readonly", "writeback"]
+        :return: None
+        """
+        # Check the input types and values
+        validator(value=cache_pool, valid_type=six.string_types)
+        validator(value=mode, valid_type=six.string_types, valid_range=["readonly", "writeback"])
+
+        check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', self.name, cache_pool])
+        check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, mode])
+        check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', self.name, cache_pool])
+        check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom'])
+
+    def remove_cache_tier(self, cache_pool):
+        """
+        Removes a cache tier from Ceph.  Flushes all dirty objects from writeback pools and waits for that to complete.
+        :param cache_pool: six.string_types.  The cache tier pool name to remove.
+        :return: None
+        """
+        # read-only is easy, writeback is much harder
+        mode = get_cache_mode(self.service, cache_pool)
+        version = ceph_version()
+        if mode == 'readonly':
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none'])
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
+
+        elif mode == 'writeback':
+            pool_forward_cmd = ['ceph', '--id', self.service, 'osd', 'tier',
+                                'cache-mode', cache_pool, 'forward']
+            if version >= '10.1':
+                # Jewel added a mandatory flag
+                pool_forward_cmd.append('--yes-i-really-mean-it')
+
+            check_call(pool_forward_cmd)
+            # Flush the cache and wait for it to return
+            check_call(['rados', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all'])
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name])
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
+
+    def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT):
+        """Return the number of placement groups to use when creating the pool.
+
+        Returns the number of placement groups which should be specified when
+        creating the pool. This is based upon the calculation guidelines
+        provided by the Ceph Placement Group Calculator (located online at
+        http://ceph.com/pgcalc/).
+
+        The number of placement groups are calculated using the following:
+
+            (Target PGs per OSD) * (OSD #) * (%Data)
+            ----------------------------------------
+                         (Pool size)
+
+        Per the upstream guidelines, the OSD # should really be considered
+        based on the number of OSDs which are eligible to be selected by the
+        pool. Since the pool creation doesn't specify any of CRUSH set rules,
+        the default rule will be dependent upon the type of pool being
+        created (replicated or erasure).
+
+        This code makes no attempt to determine the number of OSDs which can be
+        selected for the specific rule, rather it is left to the user to tune
+        in the form of 'expected-osd-count' config option.
+
+        :param pool_size: int. pool_size is either the number of replicas for
+            replicated pools or the K+M sum for erasure coded pools
+        :param percent_data: float. the percentage of data that is expected to
+            be contained in the pool for the specific OSD set. Default value
+            is to assume 10% of the data is for this pool, which is a
+            relatively low % of the data but allows for the pg_num to be
+            increased. NOTE: the default is primarily to handle the scenario
+            where related charms requiring pools has not been upgraded to
+            include an update to indicate their relative usage of the pools.
+        :return: int.  The number of pgs to use.
+        """
+
+        # Note: This calculation follows the approach that is provided
+        # by the Ceph PG Calculator located at http://ceph.com/pgcalc/.
+        validator(value=pool_size, valid_type=int)
+
+        # Ensure that percent data is set to something - even with a default
+        # it can be set to None, which would wreak havoc below.
+        if percent_data is None:
+            percent_data = DEFAULT_POOL_WEIGHT
+
+        # If the expected-osd-count is specified, then use the max between
+        # the expected-osd-count and the actual osd_count
+        osd_list = get_osds(self.service)
+        expected = config('expected-osd-count') or 0
+
+        if osd_list:
+            osd_count = max(expected, len(osd_list))
+
+            # Log a message to provide some insight if the calculations claim
+            # to be off because someone is setting the expected count and
+            # there are more OSDs in reality. Try to make a proper guess
+            # based upon the cluster itself.
+            if expected and osd_count != expected:
+                log("Found more OSDs than provided expected count. "
+                    "Using the actual count instead", INFO)
+        elif expected:
+            # Use the expected-osd-count in older ceph versions to allow for
+            # a more accurate pg calculations
+            osd_count = expected
+        else:
+            # NOTE(james-page): Default to 200 for older ceph versions
+            # which don't support OSD query from cli
+            return LEGACY_PG_COUNT
+
+        percent_data /= 100.0
+        target_pgs_per_osd = config('pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET
+        num_pg = (target_pgs_per_osd * osd_count * percent_data) // pool_size
+
+        # NOTE: ensure a sane minimum number of PGS otherwise we don't get any
+        #       reasonable data distribution in minimal OSD configurations
+        if num_pg < DEFAULT_MINIMUM_PGS:
+            num_pg = DEFAULT_MINIMUM_PGS
+
+        # The CRUSH algorithm has a slight optimization for placement groups
+        # with powers of 2 so find the nearest power of 2. If the nearest
+        # power of 2 is more than 25% below the original value, the next
+        # highest value is used. To do this, find the nearest power of 2 such
+        # that 2^n <= num_pg, check to see if its within the 25% tolerance.
+        exponent = math.floor(math.log(num_pg, 2))
+        nearest = 2 ** exponent
+        if (num_pg - nearest) > (num_pg * 0.25):
+            # Choose the next highest power of 2 since the nearest is more
+            # than 25% below the original value.
+            return int(nearest * 2)
+        else:
+            return int(nearest)
+
+
+class ReplicatedPool(Pool):
+    def __init__(self, service, name, pg_num=None, replicas=2,
+                 percent_data=10.0):
+        super(ReplicatedPool, self).__init__(service=service, name=name)
+        self.replicas = replicas
+        if pg_num:
+            # Since the number of placement groups were specified, ensure
+            # that there aren't too many created.
+            max_pgs = self.get_pgs(self.replicas, 100.0)
+            self.pg_num = min(pg_num, max_pgs)
+        else:
+            self.pg_num = self.get_pgs(self.replicas, percent_data)
+
+    def create(self):
+        if not pool_exists(self.service, self.name):
+            # Create it
+            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create',
+                   self.name, str(self.pg_num)]
+            try:
+                check_call(cmd)
+                # Set the pool replica size
+                update_pool(client=self.service,
+                            pool=self.name,
+                            settings={'size': str(self.replicas)})
+            except CalledProcessError:
+                raise
+
+
+# Default jerasure erasure coded pool
+class ErasurePool(Pool):
+    def __init__(self, service, name, erasure_code_profile="default",
+                 percent_data=10.0):
+        super(ErasurePool, self).__init__(service=service, name=name)
+        self.erasure_code_profile = erasure_code_profile
+        self.percent_data = percent_data
+
+    def create(self):
+        if not pool_exists(self.service, self.name):
+            # Try to find the erasure profile information in order to properly
+            # size the number of placement groups. The size of an erasure
+            # coded placement group is calculated as k+m.
+            erasure_profile = get_erasure_profile(self.service,
+                                                  self.erasure_code_profile)
+
+            # Check for errors
+            if erasure_profile is None:
+                msg = ("Failed to discover erasure profile named "
+                       "{}".format(self.erasure_code_profile))
+                log(msg, level=ERROR)
+                raise PoolCreationError(msg)
+            if 'k' not in erasure_profile or 'm' not in erasure_profile:
+                # Error
+                msg = ("Unable to find k (data chunks) or m (coding chunks) "
+                       "in erasure profile {}".format(erasure_profile))
+                log(msg, level=ERROR)
+                raise PoolCreationError(msg)
+
+            k = int(erasure_profile['k'])
+            m = int(erasure_profile['m'])
+            pgs = self.get_pgs(k + m, self.percent_data)
+            # Create it
+            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create',
+                   self.name, str(pgs), str(pgs),
+                   'erasure', self.erasure_code_profile]
+            try:
+                check_call(cmd)
+            except CalledProcessError:
+                raise
+
+    """Get an existing erasure code profile if it already exists.
+       Returns json formatted output"""
+
+
+def get_mon_map(service):
+    """
+    Returns the current monitor map.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :return: json string. :raise: ValueError if the monmap fails to parse.
+      Also raises CalledProcessError if our ceph command fails
+    """
+    try:
+        mon_status = check_output(
+            ['ceph', '--id', service,
+             'mon_status', '--format=json'])
+        try:
+            return json.loads(mon_status)
+        except ValueError as v:
+            log("Unable to parse mon_status json: {}. Error: {}".format(
+                mon_status, v.message))
+            raise
+    except CalledProcessError as e:
+        log("mon_status command failed with message: {}".format(
+            e.message))
+        raise
+
+
+def hash_monitor_names(service):
+    """
+    Uses the get_mon_map() function to get information about the monitor
+    cluster.
+    Hash the name of each monitor.  Return a sorted list of monitor hashes
+    in an ascending order.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :rtype : dict.   json dict of monitor name, ip address and rank
+    example: {
+        'name': 'ip-172-31-13-165',
+        'rank': 0,
+        'addr': '172.31.13.165:6789/0'}
+    """
+    try:
+        hash_list = []
+        monitor_list = get_mon_map(service=service)
+        if monitor_list['monmap']['mons']:
+            for mon in monitor_list['monmap']['mons']:
+                hash_list.append(
+                    hashlib.sha224(mon['name'].encode('utf-8')).hexdigest())
+            return sorted(hash_list)
+        else:
+            return None
+    except (ValueError, CalledProcessError):
+        raise
+
+
+def monitor_key_delete(service, key):
+    """
+    Delete a key and value pair from the monitor cluster
+    :param service: six.string_types. The Ceph user name to run the command under
+    Deletes a key value pair on the monitor cluster.
+    :param key: six.string_types.  The key to delete.
+    """
+    try:
+        check_output(
+            ['ceph', '--id', service,
+             'config-key', 'del', str(key)])
+    except CalledProcessError as e:
+        log("Monitor config-key put failed with message: {}".format(
+            e.output))
+        raise
+
+
+def monitor_key_set(service, key, value):
+    """
+    Sets a key value pair on the monitor cluster.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param key: six.string_types.  The key to set.
+    :param value: The value to set.  This will be converted to a string
+        before setting
+    """
+    try:
+        check_output(
+            ['ceph', '--id', service,
+             'config-key', 'put', str(key), str(value)])
+    except CalledProcessError as e:
+        log("Monitor config-key put failed with message: {}".format(
+            e.output))
+        raise
+
+
+def monitor_key_get(service, key):
+    """
+    Gets the value of an existing key in the monitor cluster.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param key: six.string_types.  The key to search for.
+    :return: Returns the value of that key or None if not found.
+    """
+    try:
+        output = check_output(
+            ['ceph', '--id', service,
+             'config-key', 'get', str(key)])
+        return output
+    except CalledProcessError as e:
+        log("Monitor config-key get failed with message: {}".format(
+            e.output))
+        return None
+
+
+def monitor_key_exists(service, key):
+    """
+    Searches for the existence of a key in the monitor cluster.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param key: six.string_types.  The key to search for
+    :return: Returns True if the key exists, False if not and raises an
+     exception if an unknown error occurs. :raise: CalledProcessError if
+     an unknown error occurs
+    """
+    try:
+        check_call(
+            ['ceph', '--id', service,
+             'config-key', 'exists', str(key)])
+        # I can return true here regardless because Ceph returns
+        # ENOENT if the key wasn't found
+        return True
+    except CalledProcessError as e:
+        if e.returncode == errno.ENOENT:
+            return False
+        else:
+            log("Unknown error from ceph config-get exists: {} {}".format(
+                e.returncode, e.output))
+            raise
+
+
+def get_erasure_profile(service, name):
+    """
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param name:
+    :return:
+    """
+    try:
+        out = check_output(['ceph', '--id', service,
+                            'osd', 'erasure-code-profile', 'get',
+                            name, '--format=json'])
+        return json.loads(out)
+    except (CalledProcessError, OSError, ValueError):
+        return None
+
+
+def pool_set(service, pool_name, key, value):
+    """
+    Sets a value for a RADOS pool in ceph.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :param key: six.string_types
+    :param value:
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, value]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def snapshot_pool(service, pool_name, snapshot_name):
+    """
+    Snapshots a RADOS pool in ceph.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :param snapshot_name: six.string_types
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, snapshot_name]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def remove_pool_snapshot(service, pool_name, snapshot_name):
+    """
+    Remove a snapshot from a RADOS pool in ceph.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :param snapshot_name: six.string_types
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, snapshot_name]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+# max_bytes should be an int or long
+def set_pool_quota(service, pool_name, max_bytes):
+    """
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :param max_bytes: int or long
+    :return: None.  Can raise CalledProcessError
+    """
+    # Set a byte quota on a RADOS pool in ceph.
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name,
+           'max_bytes', str(max_bytes)]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def remove_pool_quota(service, pool_name):
+    """
+    Set a byte quota on a RADOS pool in ceph.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0']
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def remove_erasure_profile(service, profile_name):
+    """
+    Create a new erasure code profile if one does not already exist for it.  Updates
+    the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/
+    for more details
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param profile_name: six.string_types
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm',
+           profile_name]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure',
+                           failure_domain='host',
+                           data_chunks=2, coding_chunks=1,
+                           locality=None, durability_estimator=None):
+    """
+    Create a new erasure code profile if one does not already exist for it.  Updates
+    the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/
+    for more details
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param profile_name: six.string_types
+    :param erasure_plugin_name: six.string_types
+    :param failure_domain: six.string_types.  One of ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region',
+        'room', 'root', 'row'])
+    :param data_chunks: int
+    :param coding_chunks: int
+    :param locality: int
+    :param durability_estimator: int
+    :return: None.  Can raise CalledProcessError
+    """
+    # Ensure this failure_domain is allowed by Ceph
+    validator(failure_domain, six.string_types,
+              ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', 'room', 'root', 'row'])
+
+    cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'set', profile_name,
+           'plugin=' + erasure_plugin_name, 'k=' + str(data_chunks), 'm=' + str(coding_chunks),
+           'ruleset_failure_domain=' + failure_domain]
+    if locality is not None and durability_estimator is not None:
+        raise ValueError("create_erasure_profile should be called with k, m and one of l or c but not both.")
+
+    # Add plugin specific information
+    if locality is not None:
+        # For local erasure codes
+        cmd.append('l=' + str(locality))
+    if durability_estimator is not None:
+        # For Shec erasure codes
+        cmd.append('c=' + str(durability_estimator))
+
+    if erasure_profile_exists(service, profile_name):
+        cmd.append('--force')
+
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def rename_pool(service, old_name, new_name):
+    """
+    Rename a Ceph pool from old_name to new_name
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param old_name: six.string_types
+    :param new_name: six.string_types
+    :return: None
+    """
+    validator(value=old_name, valid_type=six.string_types)
+    validator(value=new_name, valid_type=six.string_types)
+
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'rename', old_name, new_name]
+    check_call(cmd)
+
+
+def erasure_profile_exists(service, name):
+    """
+    Check to see if an Erasure code profile already exists.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param name: six.string_types
+    :return: int or None
+    """
+    validator(value=name, valid_type=six.string_types)
+    try:
+        check_call(['ceph', '--id', service,
+                    'osd', 'erasure-code-profile', 'get',
+                    name])
+        return True
+    except CalledProcessError:
+        return False
+
+
+def get_cache_mode(service, pool_name):
+    """
+    Find the current caching mode of the pool_name given.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :return: int or None
+    """
+    validator(value=service, valid_type=six.string_types)
+    validator(value=pool_name, valid_type=six.string_types)
+    out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json'])
+    try:
+        osd_json = json.loads(out)
+        for pool in osd_json['pools']:
+            if pool['pool_name'] == pool_name:
+                return pool['cache_mode']
+        return None
+    except ValueError:
+        raise
+
+
+def pool_exists(service, name):
+    """Check to see if a RADOS pool already exists."""
+    try:
+        out = check_output(['rados', '--id', service,
+                            'lspools']).decode('UTF-8')
+    except CalledProcessError:
+        return False
+
+    return name in out.split()
+
+
+def get_osds(service):
+    """Return a list of all Ceph Object Storage Daemons currently in the
+    cluster.
+    """
+    version = ceph_version()
+    if version and version >= '0.56':
+        return json.loads(check_output(['ceph', '--id', service,
+                                        'osd', 'ls',
+                                        '--format=json']).decode('UTF-8'))
+
+    return None
+
 
 def install():
     """Basic Ceph client installation."""
@@ -101,53 +747,37 @@ def create_rbd_image(service, pool, image, sizemb):
     check_call(cmd)
 
 
-def pool_exists(service, name):
+def update_pool(client, pool, settings):
-    """Check to see if a RADOS pool already exists."""
+    cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool]
-    try:
+    for k, v in six.iteritems(settings):
-        out = check_output(['rados', '--id', service,
+        cmd.append(k)
-                            'lspools']).decode('UTF-8')
+        cmd.append(v)
-    except CalledProcessError:
-        return False
 
-    return name in out
+    check_call(cmd)
 
 
-def get_osds(service):
+def create_pool(service, name, replicas=3, pg_num=None):
-    """Return a list of all Ceph Object Storage Daemons currently in the
-    cluster.
-    """
-    version = ceph_version()
-    if version and version >= '0.56':
-        return json.loads(check_output(['ceph', '--id', service,
-                                        'osd', 'ls',
-                                        '--format=json']).decode('UTF-8'))
-
-    return None
-
-
-def create_pool(service, name, replicas=3):
     """Create a new RADOS pool."""
     if pool_exists(service, name):
         log("Ceph pool {} already exists, skipping creation".format(name),
             level=WARNING)
         return
 
-    # Calculate the number of placement groups based
+    if not pg_num:
-    # on upstream recommended best practices.
+        # Calculate the number of placement groups based
-    osds = get_osds(service)
+        # on upstream recommended best practices.
-    if osds:
+        osds = get_osds(service)
-        pgnum = (len(osds) * 100 // replicas)
+        if osds:
-    else:
+            pg_num = (len(osds) * 100 // replicas)
-        # NOTE(james-page): Default to 200 for older ceph versions
+        else:
-        # which don't support OSD query from cli
+            # NOTE(james-page): Default to 200 for older ceph versions
-        pgnum = 200
+            # which don't support OSD query from cli
+            pg_num = 200
 
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)]
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)]
     check_call(cmd)
 
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size',
+    update_pool(service, name, settings={'size': str(replicas)})
-           str(replicas)]
-    check_call(cmd)
 
 
 def delete_pool(service, name):
@@ -202,10 +832,10 @@ def create_key_file(service, key):
     log('Created new keyfile at %s.' % keyfile, level=INFO)
 
 
-def get_ceph_nodes():
+def get_ceph_nodes(relation='ceph'):
-    """Query named relation 'ceph' to determine current nodes."""
+    """Query named relation to determine current nodes."""
     hosts = []
-    for r_id in relation_ids('ceph'):
+    for r_id in relation_ids(relation):
         for unit in related_units(r_id):
             hosts.append(relation_get('private-address', unit=unit, rid=r_id))
 
@@ -357,18 +987,20 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
             service_start(svc)
 
 
-def ensure_ceph_keyring(service, user=None, group=None):
+def ensure_ceph_keyring(service, user=None, group=None,
+                        relation='ceph', key=None):
     """Ensures a ceph keyring is created for a named service and optionally
     ensures user and group ownership.
 
-    Returns False if no ceph key is available in relation state.
+    @returns boolean: Flag to indicate whether a key was successfully written
+                      to disk based on either relation data or a supplied key
     """
-    key = None
+    if not key:
-    for rid in relation_ids('ceph'):
+        for rid in relation_ids(relation):
-        for unit in related_units(rid):
+            for unit in related_units(rid):
-            key = relation_get('key', rid=rid, unit=unit)
+                key = relation_get('key', rid=rid, unit=unit)
-            if key:
+                if key:
-                break
+                    break
 
     if not key:
         return False
@@ -405,6 +1037,7 @@ class CephBrokerRq(object):
 
     The API is versioned and defaults to version 1.
     """
+
     def __init__(self, api_version=1, request_id=None):
         self.api_version = api_version
         if request_id:
@@ -413,9 +1046,33 @@ class CephBrokerRq(object):
             self.request_id = str(uuid.uuid1())
         self.ops = []
 
-    def add_op_create_pool(self, name, replica_count=3):
+    def add_op_request_access_to_group(self, name, namespace=None,
+                                       permission=None, key_name=None):
+        """
+        Adds the requested permissions to the current service's Ceph key,
+        allowing the key to access only the specified pools
+        """
+        self.ops.append({'op': 'add-permissions-to-key', 'group': name,
+                         'namespace': namespace, 'name': key_name or service_name(),
+                         'group-permission': permission})
+
+    def add_op_create_pool(self, name, replica_count=3, pg_num=None,
+                           weight=None, group=None, namespace=None):
+        """Adds an operation to create a pool.
+
+        @param pg_num setting:  optional setting. If not provided, this value
+        will be calculated by the broker based on how many OSDs are in the
+        cluster at the time of creation. Note that, if provided, this value
+        will be capped at the current available maximum.
+        @param weight: the percentage of data the pool makes up
+        """
+        if pg_num and weight:
+            raise ValueError('pg_num and weight are mutually exclusive')
+
         self.ops.append({'op': 'create-pool', 'name': name,
-                         'replicas': replica_count})
+                         'replicas': replica_count, 'pg_num': pg_num,
+                         'weight': weight, 'group': group,
+                         'group-namespace': namespace})
 
     def set_ops(self, ops):
         """Set request ops to provided value.
@@ -433,8 +1090,8 @@ class CephBrokerRq(object):
     def _ops_equal(self, other):
         if len(self.ops) == len(other.ops):
             for req_no in range(0, len(self.ops)):
-                for key in ['replicas', 'name', 'op']:
+                for key in ['replicas', 'name', 'op', 'pg_num', 'weight']:
-                    if self.ops[req_no][key] != other.ops[req_no][key]:
+                    if self.ops[req_no].get(key) != other.ops[req_no].get(key):
                         return False
         else:
             return False
@@ -540,7 +1197,7 @@ def get_previous_request(rid):
     return request
 
 
-def get_request_states(request):
+def get_request_states(request, relation='ceph'):
     """Return a dict of requests per relation id with their corresponding
        completion state.
 
@@ -552,7 +1209,7 @@ def get_request_states(request):
     """
     complete = []
     requests = {}
-    for rid in relation_ids('ceph'):
+    for rid in relation_ids(relation):
         complete = False
         previous_request = get_previous_request(rid)
         if request == previous_request:
@@ -570,14 +1227,14 @@ def get_request_states(request):
     return requests
 
 
-def is_request_sent(request):
+def is_request_sent(request, relation='ceph'):
     """Check to see if a functionally equivalent request has already been sent
 
     Returns True if a similair request has been sent
 
     @param request: A CephBrokerRq object
     """
-    states = get_request_states(request)
+    states = get_request_states(request, relation=relation)
     for rid in states.keys():
         if not states[rid]['sent']:
             return False
@@ -585,7 +1242,7 @@ def is_request_sent(request):
     return True
 
 
-def is_request_complete(request):
+def is_request_complete(request, relation='ceph'):
     """Check to see if a functionally equivalent request has already been
     completed
 
@@ -593,7 +1250,7 @@ def is_request_complete(request):
 
     @param request: A CephBrokerRq object
     """
-    states = get_request_states(request)
+    states = get_request_states(request, relation=relation)
     for rid in states.keys():
         if not states[rid]['complete']:
             return False
@@ -643,15 +1300,54 @@ def get_broker_rsp_key():
     return 'broker-rsp-' + local_unit().replace('/', '-')
 
 
-def send_request_if_needed(request):
+def send_request_if_needed(request, relation='ceph'):
     """Send broker request if an equivalent request has not already been sent
 
     @param request: A CephBrokerRq object
     """
-    if is_request_sent(request):
+    if is_request_sent(request, relation=relation):
         log('Request already sent but not complete, not sending new request',
             level=DEBUG)
     else:
-        for rid in relation_ids('ceph'):
+        for rid in relation_ids(relation):
             log('Sending request {}'.format(request.request_id), level=DEBUG)
             relation_set(relation_id=rid, broker_req=request.request)
+
+
+class CephConfContext(object):
+    """Ceph config (ceph.conf) context.
+
+    Supports user-provided Ceph configuration settings. Use can provide a
+    dictionary as the value for the config-flags charm option containing
+    Ceph configuration settings keyede by their section in ceph.conf.
+    """
+    def __init__(self, permitted_sections=None):
+        self.permitted_sections = permitted_sections or []
+
+    def __call__(self):
+        conf = config('config-flags')
+        if not conf:
+            return {}
+
+        conf = config_flags_parser(conf)
+        if type(conf) != dict:
+            log("Provided config-flags is not a dictionary - ignoring",
+                level=WARNING)
+            return {}
+
+        permitted = self.permitted_sections
+        if permitted:
+            diff = set(conf.keys()).difference(set(permitted))
+            if diff:
+                log("Config-flags contains invalid keys '%s' - they will be "
+                    "ignored" % (', '.join(diff)), level=WARNING)
+
+        ceph_conf = {}
+        for key in conf:
+            if permitted and key not in permitted:
+                log("Ignoring key '%s'" % key, level=WARNING)
+                continue
+
+            ceph_conf[key] = conf[key]
+
+        return ceph_conf

charmhelpers/contrib/storage/linux/loopback.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import re
@@ -76,3 +74,13 @@ def ensure_loopback_device(path, size):
         check_call(cmd)
 
     return create_loopback(path)
+
+
+def is_mapped_loopback_device(device):
+    """
+    Checks if a given device name is an existing/mapped loopback device.
+    :param device: str: Full path to the device (eg, /dev/loop1).
+    :returns: str: Path to the backing file if is a loopback device
+    empty string otherwise
+    """
+    return loopback_devices().get(device, "")

charmhelpers/contrib/storage/linux/lvm.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 from subprocess import (
     CalledProcessError,

charmhelpers/contrib/storage/linux/utils.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import re
@@ -64,8 +62,8 @@ def is_device_mounted(device):
     :returns: boolean: True if the path represents a mounted device, False if
         it doesn't.
     '''
-    is_partition = bool(re.search(r".*[0-9]+\b", device))
+    try:
-    out = check_output(['mount']).decode('UTF-8')
+        out = check_output(['lsblk', '-P', device]).decode('UTF-8')
-    if is_partition:
+    except:
-        return bool(re.search(device + r"\b", out))
+        return False
-    return bool(re.search(device + r"[0-9]*\b", out))
+    return bool(re.search(r'MOUNTPOINT=".+"', out))

charmhelpers/core/__init__.py

@@ -1,15 +1,13 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

charmhelpers/core/decorators.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 #
 # Copyright 2014 Canonical Ltd.

charmhelpers/core/files.py

@@ -3,19 +3,17 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 __author__ = 'Jorge Niedbalski <niedbalski@ubuntu.com>'
 

charmhelpers/core/fstab.py

@@ -3,19 +3,17 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import io
 import os

charmhelpers/core/hookenv.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 "Interactions with the Juju environment"
 # Copyright 2013 Canonical Ltd.
@@ -334,6 +332,8 @@ def config(scope=None):
     config_cmd_line = ['config-get']
     if scope is not None:
         config_cmd_line.append(scope)
+    else:
+        config_cmd_line.append('--all')
     config_cmd_line.append('--format=json')
     try:
         config_data = json.loads(
@@ -490,6 +490,19 @@ def relation_types():
     return rel_types
 
 
+@cached
+def peer_relation_id():
+    '''Get the peers relation id if a peers relation has been joined, else None.'''
+    md = metadata()
+    section = md.get('peers')
+    if section:
+        for key in section:
+            relids = relation_ids(key)
+            if relids:
+                return relids[0]
+    return None
+
+
 @cached
 def relation_to_interface(relation_name):
     """
@@ -504,12 +517,12 @@ def relation_to_interface(relation_name):
 def relation_to_role_and_interface(relation_name):
     """
     Given the name of a relation, return the role and the name of the interface
-    that relation uses (where role is one of ``provides``, ``requires``, or ``peer``).
+    that relation uses (where role is one of ``provides``, ``requires``, or ``peers``).
 
     :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
     """
     _metadata = metadata()
-    for role in ('provides', 'requires', 'peer'):
+    for role in ('provides', 'requires', 'peers'):
         interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
         if interface:
             return role, interface
@@ -521,7 +534,7 @@ def role_and_interface_to_relations(role, interface_name):
     """
     Given a role and interface name, return a list of relation names for the
     current charm that use that interface under that role (where role is one
-    of ``provides``, ``requires``, or ``peer``).
+    of ``provides``, ``requires``, or ``peers``).
 
     :returns: A list of relation names.
     """
@@ -542,7 +555,7 @@ def interface_to_relations(interface_name):
     :returns: A list of relation names.
     """
     results = []
-    for role in ('provides', 'requires', 'peer'):
+    for role in ('provides', 'requires', 'peers'):
         results.extend(role_and_interface_to_relations(role, interface_name))
     return results
 
@@ -603,6 +616,20 @@ def close_port(port, protocol="TCP"):
     subprocess.check_call(_args)
 
 
+def open_ports(start, end, protocol="TCP"):
+    """Opens a range of service network ports"""
+    _args = ['open-port']
+    _args.append('{}-{}/{}'.format(start, end, protocol))
+    subprocess.check_call(_args)
+
+
+def close_ports(start, end, protocol="TCP"):
+    """Close a range of service network ports"""
+    _args = ['close-port']
+    _args.append('{}-{}/{}'.format(start, end, protocol))
+    subprocess.check_call(_args)
+
+
 @cached
 def unit_get(attribute):
     """Get the unit ID for the remote unit"""
@@ -624,7 +651,7 @@ def unit_private_ip():
 
 
 @cached
-def storage_get(attribute="", storage_id=""):
+def storage_get(attribute=None, storage_id=None):
     """Get storage attributes"""
     _args = ['storage-get', '--format=json']
     if storage_id:
@@ -638,7 +665,7 @@ def storage_get(attribute="", storage_id=""):
 
 
 @cached
-def storage_list(storage_name=""):
+def storage_list(storage_name=None):
     """List the storage IDs for the unit"""
     _args = ['storage-list', '--format=json']
     if storage_name:
@@ -820,6 +847,7 @@ def status_get():
 
 def translate_exc(from_exc, to_exc):
     def inner_translate_exc1(f):
+        @wraps(f)
         def inner_translate_exc2(*args, **kwargs):
             try:
                 return f(*args, **kwargs)
@@ -831,6 +859,20 @@ def translate_exc(from_exc, to_exc):
     return inner_translate_exc1
 
 
+def application_version_set(version):
+    """Charm authors may trigger this command from any hook to output what
+    version of the application is running. This could be a package version,
+    for instance postgres version 9.5. It could also be a build number or
+    version control revision identifier, for instance git sha 6fb7ba68. """
+
+    cmd = ['application-version-set']
+    cmd.append(version)
+    try:
+        subprocess.check_call(cmd)
+    except OSError:
+        log("Application Version: {}".format(version))
+
+
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
 def is_leader():
     """Does the current unit hold the juju leadership
@@ -864,6 +906,58 @@ def leader_set(settings=None, **kwargs):
     subprocess.check_call(cmd)
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_register(ptype, klass, pid):
+    """ is used while a hook is running to let Juju know that a
+        payload has been started."""
+    cmd = ['payload-register']
+    for x in [ptype, klass, pid]:
+        cmd.append(x)
+    subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_unregister(klass, pid):
+    """ is used while a hook is running to let Juju know
+    that a payload has been manually stopped. The <class> and <id> provided
+    must match a payload that has been previously registered with juju using
+    payload-register."""
+    cmd = ['payload-unregister']
+    for x in [klass, pid]:
+        cmd.append(x)
+    subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_status_set(klass, pid, status):
+    """is used to update the current status of a registered payload.
+    The <class> and <id> provided must match a payload that has been previously
+    registered with juju using payload-register. The <status> must be one of the
+    follow: starting, started, stopping, stopped"""
+    cmd = ['payload-status-set']
+    for x in [klass, pid, status]:
+        cmd.append(x)
+    subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def resource_get(name):
+    """used to fetch the resource path of the given name.
+
+    <name> must match a name of defined resource in metadata.yaml
+
+    returns either a path or False if resource not available
+    """
+    if not name:
+        return False
+
+    cmd = ['resource-get', name]
+    try:
+        return subprocess.check_output(cmd).decode('UTF-8')
+    except subprocess.CalledProcessError:
+        return False
+
+
 @cached
 def juju_version():
     """Full version string (eg. '1.23.3.1-trusty-amd64')"""
@@ -928,3 +1022,47 @@ def _run_atexit():
     for callback, args, kwargs in reversed(_atexit):
         callback(*args, **kwargs)
     del _atexit[:]
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get_primary_address(binding):
+    '''
+    Retrieve the primary network address for a named binding
+
+    :param binding: string. The name of a relation of extra-binding
+    :return: string. The primary IP address for the named binding
+    :raise: NotImplementedError if run on Juju < 2.0
+    '''
+    cmd = ['network-get', '--primary-address', binding]
+    return subprocess.check_output(cmd).decode('UTF-8').strip()
+
+
+def add_metric(*args, **kwargs):
+    """Add metric values. Values may be expressed with keyword arguments. For
+    metric names containing dashes, these may be expressed as one or more
+    'key=value' positional arguments. May only be called from the collect-metrics
+    hook."""
+    _args = ['add-metric']
+    _kvpairs = []
+    _kvpairs.extend(args)
+    _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()])
+    _args.extend(sorted(_kvpairs))
+    try:
+        subprocess.check_call(_args)
+        return
+    except EnvironmentError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs))
+    log(log_message, level='INFO')
+
+
+def meter_status():
+    """Get the meter status, if running in the meter-status-changed hook."""
+    return os.environ.get('JUJU_METER_STATUS')
+
+
+def meter_info():
+    """Get the meter status information, if running in the meter-status-changed
+    hook."""
+    return os.environ.get('JUJU_METER_INFO')

charmhelpers/core/host.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 """Tools for working with the host system"""
 # Copyright 2012 Canonical Ltd.
@@ -30,47 +28,171 @@ import random
 import string
 import subprocess
 import hashlib
-from contextlib import contextmanager
+import functools
-from collections import OrderedDict
+import itertools
-
 import six
 
+from contextlib import contextmanager
+from collections import OrderedDict
 from .hookenv import log
 from .fstab import Fstab
+from charmhelpers.osplatform import get_platform
+
+__platform__ = get_platform()
+if __platform__ == "ubuntu":
+    from charmhelpers.core.host_factory.ubuntu import (
+        service_available,
+        add_new_group,
+        lsb_release,
+        cmp_pkgrevno,
+        CompareHostReleases,
+    )  # flake8: noqa -- ignore F401 for this import
+elif __platform__ == "centos":
+    from charmhelpers.core.host_factory.centos import (
+        service_available,
+        add_new_group,
+        lsb_release,
+        cmp_pkgrevno,
+        CompareHostReleases,
+    )  # flake8: noqa -- ignore F401 for this import
+
+UPDATEDB_PATH = '/etc/updatedb.conf'
+
+def service_start(service_name, **kwargs):
+    """Start a system service.
+
+    The specified service name is managed via the system level init system.
+    Some init systems (e.g. upstart) require that additional arguments be
+    provided in order to directly control service instances whereas other init
+    systems allow for addressing instances of a service directly by name (e.g.
+    systemd).
+
+    The kwargs allow for the additional parameters to be passed to underlying
+    init systems for those systems which require/allow for them. For example,
+    the ceph-osd upstart script requires the id parameter to be passed along
+    in order to identify which running daemon should be reloaded. The follow-
+    ing example stops the ceph-osd service for instance id=4:
+
+    service_stop('ceph-osd', id=4)
+
+    :param service_name: the name of the service to stop
+    :param **kwargs: additional parameters to pass to the init system when
+                     managing services. These will be passed as key=value
+                     parameters to the init system's commandline. kwargs
+                     are ignored for systemd enabled systems.
+    """
+    return service('start', service_name, **kwargs)
 
 
-def service_start(service_name):
+def service_stop(service_name, **kwargs):
-    """Start a system service"""
+    """Stop a system service.
-    return service('start', service_name)
+
+    The specified service name is managed via the system level init system.
+    Some init systems (e.g. upstart) require that additional arguments be
+    provided in order to directly control service instances whereas other init
+    systems allow for addressing instances of a service directly by name (e.g.
+    systemd).
+
+    The kwargs allow for the additional parameters to be passed to underlying
+    init systems for those systems which require/allow for them. For example,
+    the ceph-osd upstart script requires the id parameter to be passed along
+    in order to identify which running daemon should be reloaded. The follow-
+    ing example stops the ceph-osd service for instance id=4:
+
+    service_stop('ceph-osd', id=4)
+
+    :param service_name: the name of the service to stop
+    :param **kwargs: additional parameters to pass to the init system when
+                     managing services. These will be passed as key=value
+                     parameters to the init system's commandline. kwargs
+                     are ignored for systemd enabled systems.
+    """
+    return service('stop', service_name, **kwargs)
 
 
-def service_stop(service_name):
+def service_restart(service_name, **kwargs):
-    """Stop a system service"""
+    """Restart a system service.
-    return service('stop', service_name)
 
+    The specified service name is managed via the system level init system.
+    Some init systems (e.g. upstart) require that additional arguments be
+    provided in order to directly control service instances whereas other init
+    systems allow for addressing instances of a service directly by name (e.g.
+    systemd).
 
-def service_restart(service_name):
+    The kwargs allow for the additional parameters to be passed to underlying
-    """Restart a system service"""
+    init systems for those systems which require/allow for them. For example,
+    the ceph-osd upstart script requires the id parameter to be passed along
+    in order to identify which running daemon should be restarted. The follow-
+    ing example restarts the ceph-osd service for instance id=4:
+
+    service_restart('ceph-osd', id=4)
+
+    :param service_name: the name of the service to restart
+    :param **kwargs: additional parameters to pass to the init system when
+                     managing services. These will be passed as key=value
+                     parameters to the  init system's commandline. kwargs
+                     are ignored for init systems not allowing additional
+                     parameters via the commandline (systemd).
+    """
     return service('restart', service_name)
 
 
-def service_reload(service_name, restart_on_failure=False):
+def service_reload(service_name, restart_on_failure=False, **kwargs):
     """Reload a system service, optionally falling back to restart if
-    reload fails"""
+    reload fails.
-    service_result = service('reload', service_name)
+
+    The specified service name is managed via the system level init system.
+    Some init systems (e.g. upstart) require that additional arguments be
+    provided in order to directly control service instances whereas other init
+    systems allow for addressing instances of a service directly by name (e.g.
+    systemd).
+
+    The kwargs allow for the additional parameters to be passed to underlying
+    init systems for those systems which require/allow for them. For example,
+    the ceph-osd upstart script requires the id parameter to be passed along
+    in order to identify which running daemon should be reloaded. The follow-
+    ing example restarts the ceph-osd service for instance id=4:
+
+    service_reload('ceph-osd', id=4)
+
+    :param service_name: the name of the service to reload
+    :param restart_on_failure: boolean indicating whether to fallback to a
+                               restart if the reload fails.
+    :param **kwargs: additional parameters to pass to the init system when
+                     managing services. These will be passed as key=value
+                     parameters to the  init system's commandline. kwargs
+                     are ignored for init systems not allowing additional
+                     parameters via the commandline (systemd).
+    """
+    service_result = service('reload', service_name, **kwargs)
     if not service_result and restart_on_failure:
-        service_result = service('restart', service_name)
+        service_result = service('restart', service_name, **kwargs)
     return service_result
 
 
-def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
+                  **kwargs):
     """Pause a system service.
 
-    Stop it, and prevent it from starting again at boot."""
+    Stop it, and prevent it from starting again at boot.
-    stopped = service_stop(service_name)
+
+    :param service_name: the name of the service to pause
+    :param init_dir: path to the upstart init directory
+    :param initd_dir: path to the sysv init directory
+    :param **kwargs: additional parameters to pass to the init system when
+                     managing services. These will be passed as key=value
+                     parameters to the init system's commandline. kwargs
+                     are ignored for init systems which do not support
+                     key=value arguments via the commandline.
+    """
+    stopped = True
+    if service_running(service_name, **kwargs):
+        stopped = service_stop(service_name, **kwargs)
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
-    if os.path.exists(upstart_file):
+    if init_is_systemd():
+        service('disable', service_name)
+    elif os.path.exists(upstart_file):
         override_path = os.path.join(
             init_dir, '{}.override'.format(service_name))
         with open(override_path, 'w') as fh:
@@ -78,21 +200,32 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
     elif os.path.exists(sysv_file):
         subprocess.check_call(["update-rc.d", service_name, "disable"])
     else:
-        # XXX: Support SystemD too
         raise ValueError(
-            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
+            "Unable to detect {0} as SystemD, Upstart {1} or"
+            " SysV {2}".format(
                 service_name, upstart_file, sysv_file))
     return stopped
 
 
 def service_resume(service_name, init_dir="/etc/init",
-                   initd_dir="/etc/init.d"):
+                   initd_dir="/etc/init.d", **kwargs):
     """Resume a system service.
 
-    Reenable starting again at boot. Start the service"""
+    Reenable starting again at boot. Start the service.
+
+    :param service_name: the name of the service to resume
+    :param init_dir: the path to the init dir
+    :param initd dir: the path to the initd dir
+    :param **kwargs: additional parameters to pass to the init system when
+                     managing services. These will be passed as key=value
+                     parameters to the init system's commandline. kwargs
+                     are ignored for systemd enabled systems.
+    """
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
-    if os.path.exists(upstart_file):
+    if init_is_systemd():
+        service('enable', service_name)
+    elif os.path.exists(upstart_file):
         override_path = os.path.join(
             init_dir, '{}.override'.format(service_name))
         if os.path.exists(override_path):
@@ -100,56 +233,117 @@ def service_resume(service_name, init_dir="/etc/init",
     elif os.path.exists(sysv_file):
         subprocess.check_call(["update-rc.d", service_name, "enable"])
     else:
-        # XXX: Support SystemD too
         raise ValueError(
-            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
+            "Unable to detect {0} as SystemD, Upstart {1} or"
+            " SysV {2}".format(
                 service_name, upstart_file, sysv_file))
+    started = service_running(service_name, **kwargs)
 
-    started = service_start(service_name)
+    if not started:
+        started = service_start(service_name, **kwargs)
     return started
 
 
-def service(action, service_name):
+def service(action, service_name, **kwargs):
-    """Control a system service"""
+    """Control a system service.
-    cmd = ['service', service_name, action]
+
+    :param action: the action to take on the service
+    :param service_name: the name of the service to perform th action on
+    :param **kwargs: additional params to be passed to the service command in
+                    the form of key=value.
+    """
+    if init_is_systemd():
+        cmd = ['systemctl', action, service_name]
+    else:
+        cmd = ['service', service_name, action]
+        for key, value in six.iteritems(kwargs):
+            parameter = '%s=%s' % (key, value)
+            cmd.append(parameter)
     return subprocess.call(cmd) == 0
 
 
-def service_running(service):
+_UPSTART_CONF = "/etc/init/{}.conf"
-    """Determine whether a system service is running"""
+_INIT_D_CONF = "/etc/init.d/{}"
-    try:
+
-        output = subprocess.check_output(
+
-            ['service', service, 'status'],
+def service_running(service_name, **kwargs):
-            stderr=subprocess.STDOUT).decode('UTF-8')
+    """Determine whether a system service is running.
-    except subprocess.CalledProcessError:
+
+    :param service_name: the name of the service
+    :param **kwargs: additional args to pass to the service command. This is
+                     used to pass additional key=value arguments to the
+                     service command line for managing specific instance
+                     units (e.g. service ceph-osd status id=2). The kwargs
+                     are ignored in systemd services.
+    """
+    if init_is_systemd():
+        return service('is-active', service_name)
+    else:
+        if os.path.exists(_UPSTART_CONF.format(service_name)):
+            try:
+                cmd = ['status', service_name]
+                for key, value in six.iteritems(kwargs):
+                    parameter = '%s=%s' % (key, value)
+                    cmd.append(parameter)
+                output = subprocess.check_output(cmd,
+                    stderr=subprocess.STDOUT).decode('UTF-8')
+            except subprocess.CalledProcessError:
+                return False
+            else:
+                # This works for upstart scripts where the 'service' command
+                # returns a consistent string to represent running
+                # 'start/running'
+                if ("start/running" in output or
+                        "is running" in output or
+                        "up and running" in output):
+                    return True
+        elif os.path.exists(_INIT_D_CONF.format(service_name)):
+            # Check System V scripts init script return codes
+            return service('status', service_name)
         return False
-    else:
-        if ("start/running" in output or "is running" in output):
-            return True
-        else:
-            return False
 
 
-def service_available(service_name):
+SYSTEMD_SYSTEM = '/run/systemd/system'
-    """Determine whether a system service is available"""
-    try:
-        subprocess.check_output(
-            ['service', service_name, 'status'],
-            stderr=subprocess.STDOUT).decode('UTF-8')
-    except subprocess.CalledProcessError as e:
-        return b'unrecognized service' not in e.output
-    else:
-        return True
 
 
-def adduser(username, password=None, shell='/bin/bash', system_user=False):
+def init_is_systemd():
-    """Add a user to the system"""
+    """Return True if the host system uses systemd, False otherwise."""
+    if lsb_release()['DISTRIB_CODENAME'] == 'trusty':
+        return False
+    return os.path.isdir(SYSTEMD_SYSTEM)
+
+
+def adduser(username, password=None, shell='/bin/bash',
+            system_user=False, primary_group=None,
+            secondary_groups=None, uid=None, home_dir=None):
+    """Add a user to the system.
+
+    Will log but otherwise succeed if the user already exists.
+
+    :param str username: Username to create
+    :param str password: Password for user; if ``None``, create a system user
+    :param str shell: The default shell for the user
+    :param bool system_user: Whether to create a login or system user
+    :param str primary_group: Primary group for user; defaults to username
+    :param list secondary_groups: Optional list of additional groups
+    :param int uid: UID for user being created
+    :param str home_dir: Home directory for user
+
+    :returns: The password database entry struct, as returned by `pwd.getpwnam`
+    """
     try:
         user_info = pwd.getpwnam(username)
         log('user {0} already exists!'.format(username))
+        if uid:
+            user_info = pwd.getpwuid(int(uid))
+            log('user with uid {0} already exists!'.format(uid))
     except KeyError:
         log('creating user {0}'.format(username))
         cmd = ['useradd']
+        if uid:
+            cmd.extend(['--uid', str(uid)])
+        if home_dir:
+            cmd.extend(['--home', str(home_dir)])
         if system_user or password is None:
             cmd.append('--system')
         else:
@@ -158,6 +352,16 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
                 '--shell', shell,
                 '--password', password,
             ])
+        if not primary_group:
+            try:
+                grp.getgrnam(username)
+                primary_group = username  # avoid "group exists" error
+            except KeyError:
+                pass
+        if primary_group:
+            cmd.extend(['-g', primary_group])
+        if secondary_groups:
+            cmd.extend(['-G', ','.join(secondary_groups)])
         cmd.append(username)
         subprocess.check_call(cmd)
         user_info = pwd.getpwnam(username)
@@ -174,22 +378,56 @@ def user_exists(username):
     return user_exists
 
 
-def add_group(group_name, system_group=False):
+def uid_exists(uid):
-    """Add a group to the system"""
+    """Check if a uid exists"""
+    try:
+        pwd.getpwuid(uid)
+        uid_exists = True
+    except KeyError:
+        uid_exists = False
+    return uid_exists
+
+
+def group_exists(groupname):
+    """Check if a group exists"""
+    try:
+        grp.getgrnam(groupname)
+        group_exists = True
+    except KeyError:
+        group_exists = False
+    return group_exists
+
+
+def gid_exists(gid):
+    """Check if a gid exists"""
+    try:
+        grp.getgrgid(gid)
+        gid_exists = True
+    except KeyError:
+        gid_exists = False
+    return gid_exists
+
+
+def add_group(group_name, system_group=False, gid=None):
+    """Add a group to the system
+
+    Will log but otherwise succeed if the group already exists.
+
+    :param str group_name: group to create
+    :param bool system_group: Create system group
+    :param int gid: GID for user being created
+
+    :returns: The password database entry struct, as returned by `grp.getgrnam`
+    """
     try:
         group_info = grp.getgrnam(group_name)
         log('group {0} already exists!'.format(group_name))
+        if gid:
+            group_info = grp.getgrgid(gid)
+            log('group with gid {0} already exists!'.format(gid))
     except KeyError:
         log('creating group {0}'.format(group_name))
-        cmd = ['addgroup']
+        add_new_group(group_name, system_group, gid)
-        if system_group:
-            cmd.append('--system')
-        else:
-            cmd.extend([
-                '--group',
-            ])
-        cmd.append(group_name)
-        subprocess.check_call(cmd)
         group_info = grp.getgrnam(group_name)
     return group_info
 
@@ -201,15 +439,17 @@ def add_user_to_group(username, group):
     subprocess.check_call(cmd)
 
 
-def rsync(from_path, to_path, flags='-r', options=None):
+def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
     """Replicate the contents of a path"""
     options = options or ['--delete', '--executability']
     cmd = ['/usr/bin/rsync', flags]
+    if timeout:
+        cmd = ['timeout', str(timeout)] + cmd
     cmd.extend(options)
     cmd.append(from_path)
     cmd.append(to_path)
     log(" ".join(cmd))
-    return subprocess.check_output(cmd).decode('UTF-8').strip()
+    return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
 
 
 def symlink(source, destination):
@@ -255,14 +495,12 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
 
 
 def fstab_remove(mp):
-    """Remove the given mountpoint entry from /etc/fstab
+    """Remove the given mountpoint entry from /etc/fstab"""
-    """
     return Fstab.remove_by_mountpoint(mp)
 
 
 def fstab_add(dev, mp, fs, options=None):
-    """Adds the given device entry to the /etc/fstab file
+    """Adds the given device entry to the /etc/fstab file"""
-    """
     return Fstab.add(dev, mp, fs, options=options)
 
 
@@ -318,8 +556,7 @@ def fstab_mount(mountpoint):
 
 
 def file_hash(path, hash_type='md5'):
-    """
+    """Generate a hash checksum of the contents of 'path' or None if not found.
-    Generate a hash checksum of the contents of 'path' or None if not found.
 
     :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
                           such as md5, sha1, sha256, sha512, etc.
@@ -334,10 +571,9 @@ def file_hash(path, hash_type='md5'):
 
 
 def path_hash(path):
-    """
+    """Generate a hash checksum of all files matching 'path'. Standard
-    Generate a hash checksum of all files matching 'path'. Standard wildcards
+    wildcards like '*' and '?' are supported, see documentation for the 'glob'
-    like '*' and '?' are supported, see documentation for the 'glob' module for
+    module for more information.
-    more information.
 
     :return: dict: A { filename: hash } dictionary for all matched files.
                    Empty if none found.
@@ -349,8 +585,7 @@ def path_hash(path):
 
 
 def check_hash(path, checksum, hash_type='md5'):
-    """
+    """Validate a file using a cryptographic checksum.
-    Validate a file using a cryptographic checksum.
 
     :param str checksum: Value of the checksum used to validate the file.
     :param str hash_type: Hash algorithm used to generate `checksum`.
@@ -365,10 +600,11 @@ def check_hash(path, checksum, hash_type='md5'):
 
 
 class ChecksumError(ValueError):
+    """A class derived from Value error to indicate the checksum failed."""
     pass
 
 
-def restart_on_change(restart_map, stopstart=False):
+def restart_on_change(restart_map, stopstart=False, restart_functions=None):
     """Restart services based on configuration files changing
 
     This function is used a decorator, for example::
@@ -386,35 +622,56 @@ def restart_on_change(restart_map, stopstart=False):
     restarted if any file matching the pattern got changed, created
     or removed. Standard wildcards are supported, see documentation
     for the 'glob' module for more information.
+
+    @param restart_map: {path_file_name: [service_name, ...]
+    @param stopstart: DEFAULT false; whether to stop, start OR restart
+    @param restart_functions: nonstandard functions to use to restart services
+                              {svc: func, ...}
+    @returns result from decorated function
     """
     def wrap(f):
+        @functools.wraps(f)
         def wrapped_f(*args, **kwargs):
-            checksums = {path: path_hash(path) for path in restart_map}
+            return restart_on_change_helper(
-            f(*args, **kwargs)
+                (lambda: f(*args, **kwargs)), restart_map, stopstart,
-            restarts = []
+                restart_functions)
-            for path in restart_map:
-                if path_hash(path) != checksums[path]:
-                    restarts += restart_map[path]
-            services_list = list(OrderedDict.fromkeys(restarts))
-            if not stopstart:
-                for service_name in services_list:
-                    service('restart', service_name)
-            else:
-                for action in ['stop', 'start']:
-                    for service_name in services_list:
-                        service(action, service_name)
         return wrapped_f
     return wrap
 
 
-def lsb_release():
+def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
-    """Return /etc/lsb-release in a dict"""
+                             restart_functions=None):
-    d = {}
+    """Helper function to perform the restart_on_change function.
-    with open('/etc/lsb-release', 'r') as lsb:
+
-        for l in lsb:
+    This is provided for decorators to restart services if files described
-            k, v = l.split('=')
+    in the restart_map have changed after an invocation of lambda_f().
-            d[k.strip()] = v.strip()
+
-    return d
+    @param lambda_f: function to call.
+    @param restart_map: {file: [service, ...]}
+    @param stopstart: whether to stop, start or restart a service
+    @param restart_functions: nonstandard functions to use to restart services
+                              {svc: func, ...}
+    @returns result of lambda_f()
+    """
+    if restart_functions is None:
+        restart_functions = {}
+    checksums = {path: path_hash(path) for path in restart_map}
+    r = lambda_f()
+    # create a list of lists of the services to restart
+    restarts = [restart_map[path]
+                for path in restart_map
+                if path_hash(path) != checksums[path]]
+    # create a flat list of ordered services without duplicates from lists
+    services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts)))
+    if services_list:
+        actions = ('stop', 'start') if stopstart else ('restart',)
+        for service_name in services_list:
+            if service_name in restart_functions:
+                restart_functions[service_name](service_name)
+            else:
+                for action in actions:
+                    service(action, service_name)
+    return r
 
 
 def pwgen(length=None):
@@ -470,7 +727,7 @@ def get_bond_master(interface):
 
 
 def list_nics(nic_type=None):
-    '''Return a list of nics of given type(s)'''
+    """Return a list of nics of given type(s)"""
     if isinstance(nic_type, six.string_types):
         int_types = [nic_type]
     else:
@@ -512,12 +769,13 @@ def list_nics(nic_type=None):
 
 
 def set_nic_mtu(nic, mtu):
-    '''Set MTU on a network interface'''
+    """Set the Maximum Transmission Unit (MTU) on a network interface."""
     cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
     subprocess.check_call(cmd)
 
 
 def get_nic_mtu(nic):
+    """Return the Maximum Transmission Unit (MTU) for a network interface."""
     cmd = ['ip', 'addr', 'show', nic]
     ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
     mtu = ""
@@ -529,6 +787,7 @@ def get_nic_mtu(nic):
 
 
 def get_nic_hwaddr(nic):
+    """Return the Media Access Control (MAC) for a network interface."""
     cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
     ip_output = subprocess.check_output(cmd).decode('UTF-8')
     hwaddr = ""
@@ -538,35 +797,31 @@ def get_nic_hwaddr(nic):
     return hwaddr
 
 
-def cmp_pkgrevno(package, revno, pkgcache=None):
-    '''Compare supplied revno with the revno of the installed package
-
-    *  1 => Installed revno is greater than supplied arg
-    *  0 => Installed revno is the same as supplied arg
-    * -1 => Installed revno is less than supplied arg
-
-    This function imports apt_cache function from charmhelpers.fetch if
-    the pkgcache argument is None. Be sure to add charmhelpers.fetch if
-    you call this function, or pass an apt_pkg.Cache() instance.
-    '''
-    import apt_pkg
-    if not pkgcache:
-        from charmhelpers.fetch import apt_cache
-        pkgcache = apt_cache()
-    pkg = pkgcache[package]
-    return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
-
-
 @contextmanager
-def chdir(d):
+def chdir(directory):
+    """Change the current working directory to a different directory for a code
+    block and return the previous directory after the block exits. Useful to
+    run commands from a specificed directory.
+
+    :param str directory: The directory path to change to for this context.
+    """
     cur = os.getcwd()
     try:
-        yield os.chdir(d)
+        yield os.chdir(directory)
     finally:
         os.chdir(cur)
 
 
-def chownr(path, owner, group, follow_links=True):
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
+    """Recursively change user and group ownership of files and directories
+    in given path. Doesn't chown path itself by default, only its children.
+
+    :param str path: The string path to start changing ownership.
+    :param str owner: The owner string to use when looking up the uid.
+    :param str group: The group string to use when looking up the gid.
+    :param bool follow_links: Also follow and chown links if True
+    :param bool chowntopdir: Also chown path itself if True
+    """
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
     if follow_links:
@@ -574,7 +829,11 @@ def chownr(path, owner, group, follow_links=True):
     else:
         chown = os.lchown
 
-    for root, dirs, files in os.walk(path):
+    if chowntopdir:
+        broken_symlink = os.path.lexists(path) and not os.path.exists(path)
+        if not broken_symlink:
+            chown(path, uid, gid)
+    for root, dirs, files in os.walk(path, followlinks=follow_links):
         for name in dirs + files:
             full = os.path.join(root, name)
             broken_symlink = os.path.lexists(full) and not os.path.exists(full)
@@ -583,4 +842,81 @@ def chownr(path, owner, group, follow_links=True):
 
 
 def lchownr(path, owner, group):
+    """Recursively change user and group ownership of files and directories
+    in a given path, not following symbolic links. See the documentation for
+    'os.lchown' for more information.
+
+    :param str path: The string path to start changing ownership.
+    :param str owner: The owner string to use when looking up the uid.
+    :param str group: The group string to use when looking up the gid.
+    """
     chownr(path, owner, group, follow_links=False)
+
+
+def owner(path):
+    """Returns a tuple containing the username & groupname owning the path.
+
+    :param str path: the string path to retrieve the ownership
+    :return tuple(str, str): A (username, groupname) tuple containing the
+                             name of the user and group owning the path.
+    :raises OSError: if the specified path does not exist
+    """
+    stat = os.stat(path)
+    username = pwd.getpwuid(stat.st_uid)[0]
+    groupname = grp.getgrgid(stat.st_gid)[0]
+    return username, groupname
+
+
+def get_total_ram():
+    """The total amount of system RAM in bytes.
+
+    This is what is reported by the OS, and may be overcommitted when
+    there are multiple containers hosted on the same machine.
+    """
+    with open('/proc/meminfo', 'r') as f:
+        for line in f.readlines():
+            if line:
+                key, value, unit = line.split()
+                if key == 'MemTotal:':
+                    assert unit == 'kB', 'Unknown unit'
+                    return int(value) * 1024  # Classic, not KiB.
+        raise NotImplementedError()
+
+
+UPSTART_CONTAINER_TYPE = '/run/container_type'
+
+
+def is_container():
+    """Determine whether unit is running in a container
+
+    @return: boolean indicating if unit is in a container
+    """
+    if init_is_systemd():
+        # Detect using systemd-detect-virt
+        return subprocess.call(['systemd-detect-virt',
+                                '--container']) == 0
+    else:
+        # Detect using upstart container file marker
+        return os.path.exists(UPSTART_CONTAINER_TYPE)
+
+
+def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
+    with open(updatedb_path, 'r+') as f_id:
+        updatedb_text = f_id.read()
+        output = updatedb(updatedb_text, path)
+        f_id.seek(0)
+        f_id.write(output)
+        f_id.truncate()
+
+
+def updatedb(updatedb_text, new_path):
+    lines = [line for line in updatedb_text.split("\n")]
+    for i, line in enumerate(lines):
+        if line.startswith("PRUNEPATHS="):
+            paths_line = line.split("=")[1].replace('"', '')
+            paths = paths_line.split(" ")
+            if new_path not in paths:
+                paths.append(new_path)
+                lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
+    output = "\n".join(lines)
+    return output

charmhelpers/core/host_factory/centos.py

@@ -0,0 +1,72 @@
+import subprocess
+import yum
+import os
+
+from charmhelpers.core.strutils import BasicStringComparator
+
+
+class CompareHostReleases(BasicStringComparator):
+    """Provide comparisons of Host releases.
+
+    Use in the form of
+
+    if CompareHostReleases(release) > 'trusty':
+        # do something with mitaka
+    """
+
+    def __init__(self, item):
+        raise NotImplementedError(
+            "CompareHostReleases() is not implemented for CentOS")
+
+
+def service_available(service_name):
+    # """Determine whether a system service is available."""
+    if os.path.isdir('/run/systemd/system'):
+        cmd = ['systemctl', 'is-enabled', service_name]
+    else:
+        cmd = ['service', service_name, 'is-enabled']
+    return subprocess.call(cmd) == 0
+
+
+def add_new_group(group_name, system_group=False, gid=None):
+    cmd = ['groupadd']
+    if gid:
+        cmd.extend(['--gid', str(gid)])
+    if system_group:
+        cmd.append('-r')
+    cmd.append(group_name)
+    subprocess.check_call(cmd)
+
+
+def lsb_release():
+    """Return /etc/os-release in a dict."""
+    d = {}
+    with open('/etc/os-release', 'r') as lsb:
+        for l in lsb:
+            s = l.split('=')
+            if len(s) != 2:
+                continue
+            d[s[0].strip()] = s[1].strip()
+    return d
+
+
+def cmp_pkgrevno(package, revno, pkgcache=None):
+    """Compare supplied revno with the revno of the installed package.
+
+    *  1 => Installed revno is greater than supplied arg
+    *  0 => Installed revno is the same as supplied arg
+    * -1 => Installed revno is less than supplied arg
+
+    This function imports YumBase function if the pkgcache argument
+    is None.
+    """
+    if not pkgcache:
+        y = yum.YumBase()
+        packages = y.doPackageLists()
+        pkgcache = {i.Name: i.version for i in packages['installed']}
+    pkg = pkgcache[package]
+    if pkg > revno:
+        return 1
+    if pkg < revno:
+        return -1
+    return 0

charmhelpers/core/host_factory/ubuntu.py

@@ -0,0 +1,88 @@
+import subprocess
+
+from charmhelpers.core.strutils import BasicStringComparator
+
+
+UBUNTU_RELEASES = (
+    'lucid',
+    'maverick',
+    'natty',
+    'oneiric',
+    'precise',
+    'quantal',
+    'raring',
+    'saucy',
+    'trusty',
+    'utopic',
+    'vivid',
+    'wily',
+    'xenial',
+    'yakkety',
+    'zesty',
+)
+
+
+class CompareHostReleases(BasicStringComparator):
+    """Provide comparisons of Ubuntu releases.
+
+    Use in the form of
+
+    if CompareHostReleases(release) > 'trusty':
+        # do something with mitaka
+    """
+    _list = UBUNTU_RELEASES
+
+
+def service_available(service_name):
+    """Determine whether a system service is available"""
+    try:
+        subprocess.check_output(
+            ['service', service_name, 'status'],
+            stderr=subprocess.STDOUT).decode('UTF-8')
+    except subprocess.CalledProcessError as e:
+        return b'unrecognized service' not in e.output
+    else:
+        return True
+
+
+def add_new_group(group_name, system_group=False, gid=None):
+    cmd = ['addgroup']
+    if gid:
+        cmd.extend(['--gid', str(gid)])
+    if system_group:
+        cmd.append('--system')
+    else:
+        cmd.extend([
+            '--group',
+        ])
+    cmd.append(group_name)
+    subprocess.check_call(cmd)
+
+
+def lsb_release():
+    """Return /etc/lsb-release in a dict"""
+    d = {}
+    with open('/etc/lsb-release', 'r') as lsb:
+        for l in lsb:
+            k, v = l.split('=')
+            d[k.strip()] = v.strip()
+    return d
+
+
+def cmp_pkgrevno(package, revno, pkgcache=None):
+    """Compare supplied revno with the revno of the installed package.
+
+    *  1 => Installed revno is greater than supplied arg
+    *  0 => Installed revno is the same as supplied arg
+    * -1 => Installed revno is less than supplied arg
+
+    This function imports apt_cache function from charmhelpers.fetch if
+    the pkgcache argument is None. Be sure to add charmhelpers.fetch if
+    you call this function, or pass an apt_pkg.Cache() instance.
+    """
+    import apt_pkg
+    if not pkgcache:
+        from charmhelpers.fetch import apt_cache
+        pkgcache = apt_cache()
+    pkg = pkgcache[package]
+    return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

charmhelpers/core/hugepage.py

@@ -2,19 +2,17 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import yaml
 from charmhelpers.core import fstab
@@ -46,6 +44,8 @@ def hugepage_support(user, group='hugetlb', nr_hugepages=256,
     group_info = add_group(group)
     gid = group_info.gr_gid
     add_user_to_group(user, group)
+    if max_map_count < 2 * nr_hugepages:
+        max_map_count = 2 * nr_hugepages
     sysctl_settings = {
         'vm.nr_hugepages': nr_hugepages,
         'vm.max_map_count': max_map_count,

charmhelpers/core/kernel.py

@@ -3,29 +3,40 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
-__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+import re
+import subprocess
 
+from charmhelpers.osplatform import get_platform
 from charmhelpers.core.hookenv import (
     log,
     INFO
 )
 
-from subprocess import check_call, check_output
+__platform__ = get_platform()
-import re
+if __platform__ == "ubuntu":
+    from charmhelpers.core.kernel_factory.ubuntu import (
+        persistent_modprobe,
+        update_initramfs,
+    )  # flake8: noqa -- ignore F401 for this import
+elif __platform__ == "centos":
+    from charmhelpers.core.kernel_factory.centos import (
+        persistent_modprobe,
+        update_initramfs,
+    )  # flake8: noqa -- ignore F401 for this import
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
 
 
 def modprobe(module, persist=True):
@@ -34,11 +45,9 @@ def modprobe(module, persist=True):
 
     log('Loading kernel module %s' % module, level=INFO)
 
-    check_call(cmd)
+    subprocess.check_call(cmd)
     if persist:
-        with open('/etc/modules', 'r+') as modules:
+        persistent_modprobe(module)
-            if module not in modules.read():
-                modules.write(module)
 
 
 def rmmod(module, force=False):
@@ -48,21 +57,16 @@ def rmmod(module, force=False):
         cmd.append('-f')
     cmd.append(module)
     log('Removing kernel module %s' % module, level=INFO)
-    return check_call(cmd)
+    return subprocess.check_call(cmd)
 
 
 def lsmod():
     """Shows what kernel modules are currently loaded"""
-    return check_output(['lsmod'],
+    return subprocess.check_output(['lsmod'],
-                        universal_newlines=True)
+                                   universal_newlines=True)
 
 
 def is_module_loaded(module):
     """Checks if a kernel module is already loaded"""
     matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
     return len(matches) > 0
-
-
-def update_initramfs(version='all'):
-    """Updates an initramfs image"""
-    return check_call(["update-initramfs", "-k", version, "-u"])

charmhelpers/core/kernel_factory/centos.py

@@ -0,0 +1,17 @@
+import subprocess
+import os
+
+
+def persistent_modprobe(module):
+    """Load a kernel module and configure for auto-load on reboot."""
+    if not os.path.exists('/etc/rc.modules'):
+        open('/etc/rc.modules', 'a')
+        os.chmod('/etc/rc.modules', 111)
+    with open('/etc/rc.modules', 'r+') as modules:
+        if module not in modules.read():
+            modules.write('modprobe %s\n' % module)
+
+
+def update_initramfs(version='all'):
+    """Updates an initramfs image."""
+    return subprocess.check_call(["dracut", "-f", version])

charmhelpers/core/kernel_factory/ubuntu.py

@@ -0,0 +1,13 @@
+import subprocess
+
+
+def persistent_modprobe(module):
+    """Load a kernel module and configure for auto-load on reboot."""
+    with open('/etc/modules', 'r+') as modules:
+        if module not in modules.read():
+            modules.write(module + "\n")
+
+
+def update_initramfs(version='all'):
+    """Updates an initramfs image."""
+    return subprocess.check_call(["update-initramfs", "-k", version, "-u"])

charmhelpers/core/services/__init__.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 from .base import *  # NOQA
 from .helpers import *  # NOQA

charmhelpers/core/services/base.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import json

charmhelpers/core/services/helpers.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import yaml
@@ -243,33 +241,40 @@ class TemplateCallback(ManagerCallback):
     :param str source: The template source file, relative to
         `$CHARM_DIR/templates`
 
-    :param str target: The target to write the rendered template to
+    :param str target: The target to write the rendered template to (or None)
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
     :param partial on_change_action: functools partial to be executed when
                                      rendered file changes
+    :param jinja2 loader template_loader: A jinja2 template loader
+
+    :return str: The rendered template
     """
     def __init__(self, source, target,
                  owner='root', group='root', perms=0o444,
-                 on_change_action=None):
+                 on_change_action=None, template_loader=None):
         self.source = source
         self.target = target
         self.owner = owner
         self.group = group
         self.perms = perms
         self.on_change_action = on_change_action
+        self.template_loader = template_loader
 
     def __call__(self, manager, service_name, event_name):
         pre_checksum = ''
         if self.on_change_action and os.path.isfile(self.target):
             pre_checksum = host.file_hash(self.target)
         service = manager.get_service(service_name)
-        context = {}
+        context = {'ctx': {}}
         for ctx in service.get('required_data', []):
             context.update(ctx)
-        templating.render(self.source, self.target, context,
+            context['ctx'].update(ctx)
-                          self.owner, self.group, self.perms)
+
+        result = templating.render(self.source, self.target, context,
+                                   self.owner, self.group, self.perms,
+                                   template_loader=self.template_loader)
         if self.on_change_action:
             if pre_checksum == host.file_hash(self.target):
                 hookenv.log(
@@ -278,6 +283,8 @@ class TemplateCallback(ManagerCallback):
             else:
                 self.on_change_action()
 
+        return result
+
 
 # Convenience aliases for templates
 render_template = template = TemplateCallback

charmhelpers/core/strutils.py

@@ -3,19 +3,17 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
 import re
@@ -70,3 +68,56 @@ def bytes_from_string(value):
         msg = "Unable to interpret string value '%s' as bytes" % (value)
         raise ValueError(msg)
     return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+
+
+class BasicStringComparator(object):
+    """Provides a class that will compare strings from an iterator type object.
+    Used to provide > and < comparisons on strings that may not necessarily be
+    alphanumerically ordered.  e.g. OpenStack or Ubuntu releases AFTER the
+    z-wrap.
+    """
+
+    _list = None
+
+    def __init__(self, item):
+        if self._list is None:
+            raise Exception("Must define the _list in the class definition!")
+        try:
+            self.index = self._list.index(item)
+        except Exception:
+            raise KeyError("Item '{}' is not in list '{}'"
+                           .format(item, self._list))
+
+    def __eq__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index == self._list.index(other)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __lt__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index < self._list.index(other)
+
+    def __ge__(self, other):
+        return not self.__lt__(other)
+
+    def __gt__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index > self._list.index(other)
+
+    def __le__(self, other):
+        return not self.__gt__(other)
+
+    def __str__(self):
+        """Always give back the item at the index so it can be used in
+        comparisons like:
+
+        s_mitaka = CompareOpenStack('mitaka')
+        s_newton = CompareOpenstack('newton')
+
+        assert s_newton > s_mitaka
+
+        @returns: <string>
+        """
+        return self._list[self.index]

charmhelpers/core/sysctl.py

@@ -3,19 +3,17 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import yaml
 

charmhelpers/core/templating.py

@@ -1,33 +1,33 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import sys
 
 from charmhelpers.core import host
 from charmhelpers.core import hookenv
 
 
 def render(source, target, context, owner='root', group='root',
-           perms=0o444, templates_dir=None, encoding='UTF-8'):
+           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
     """
     Render a template.
 
     The `source` path, if not absolute, is relative to the `templates_dir`.
 
-    The `target` path should be absolute.
+    The `target` path should be absolute.  It can also be `None`, in which
+    case no file will be written.
 
     The context should be a dict containing the values to be replaced in the
     template.
@@ -36,8 +36,12 @@ def render(source, target, context, owner='root', group='root',
 
     If omitted, `templates_dir` defaults to the `templates` folder in the charm.
 
-    Note: Using this requires python-jinja2; if it is not installed, calling
+    The rendered template will be written to the file as well as being returned
-    this will attempt to use charmhelpers.fetch.apt_install to install it.
+    as a string.
+
+    Note: Using this requires python-jinja2 or python3-jinja2; if it is not
+    installed, calling this will attempt to use charmhelpers.fetch.apt_install
+    to install it.
     """
     try:
         from jinja2 import FileSystemLoader, Environment, exceptions
@@ -49,20 +53,32 @@ def render(source, target, context, owner='root', group='root',
                         'charmhelpers.fetch to install it',
                         level=hookenv.ERROR)
             raise
-        apt_install('python-jinja2', fatal=True)
+        if sys.version_info.major == 2:
+            apt_install('python-jinja2', fatal=True)
+        else:
+            apt_install('python3-jinja2', fatal=True)
         from jinja2 import FileSystemLoader, Environment, exceptions
 
-    if templates_dir is None:
+    if template_loader:
-        templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+        template_env = Environment(loader=template_loader)
-    loader = Environment(loader=FileSystemLoader(templates_dir))
+    else:
+        if templates_dir is None:
+            templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+        template_env = Environment(loader=FileSystemLoader(templates_dir))
     try:
         source = source
-        template = loader.get_template(source)
+        template = template_env.get_template(source)
     except exceptions.TemplateNotFound as e:
         hookenv.log('Could not load template %s from %s.' %
                     (source, templates_dir),
                     level=hookenv.ERROR)
         raise e
     content = template.render(context)
-    host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+    if target is not None:
-    host.write_file(target, content.encode(encoding), owner, group, perms)
+        target_dir = os.path.dirname(target)
+        if not os.path.exists(target_dir):
+            # This is a terrible default directory permission, as the file
+            # or its siblings will often contain secrets.
+            host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+        host.write_file(target, content.encode(encoding), owner, group, perms)
+    return content

charmhelpers/core/unitdata.py

@@ -3,20 +3,17 @@
 #
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
 #  Kapil Thangavelu <kapil.foss@gmail.com>

charmhelpers/fetch/__init__.py

@@ -1,32 +1,24 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import importlib
-from tempfile import NamedTemporaryFile
+from charmhelpers.osplatform import get_platform
-import time
 from yaml import safe_load
-from charmhelpers.core.host import (
-    lsb_release
-)
-import subprocess
 from charmhelpers.core.hookenv import (
     config,
     log,
 )
-import os
 
 import six
 if six.PY3:
@@ -35,71 +27,6 @@ else:
     from urlparse import urlparse, urlunparse
 
 
-CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
-deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
-"""
-PROPOSED_POCKET = """# Proposed
-deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
-"""
-CLOUD_ARCHIVE_POCKETS = {
-    # Folsom
-    'folsom': 'precise-updates/folsom',
-    'precise-folsom': 'precise-updates/folsom',
-    'precise-folsom/updates': 'precise-updates/folsom',
-    'precise-updates/folsom': 'precise-updates/folsom',
-    'folsom/proposed': 'precise-proposed/folsom',
-    'precise-folsom/proposed': 'precise-proposed/folsom',
-    'precise-proposed/folsom': 'precise-proposed/folsom',
-    # Grizzly
-    'grizzly': 'precise-updates/grizzly',
-    'precise-grizzly': 'precise-updates/grizzly',
-    'precise-grizzly/updates': 'precise-updates/grizzly',
-    'precise-updates/grizzly': 'precise-updates/grizzly',
-    'grizzly/proposed': 'precise-proposed/grizzly',
-    'precise-grizzly/proposed': 'precise-proposed/grizzly',
-    'precise-proposed/grizzly': 'precise-proposed/grizzly',
-    # Havana
-    'havana': 'precise-updates/havana',
-    'precise-havana': 'precise-updates/havana',
-    'precise-havana/updates': 'precise-updates/havana',
-    'precise-updates/havana': 'precise-updates/havana',
-    'havana/proposed': 'precise-proposed/havana',
-    'precise-havana/proposed': 'precise-proposed/havana',
-    'precise-proposed/havana': 'precise-proposed/havana',
-    # Icehouse
-    'icehouse': 'precise-updates/icehouse',
-    'precise-icehouse': 'precise-updates/icehouse',
-    'precise-icehouse/updates': 'precise-updates/icehouse',
-    'precise-updates/icehouse': 'precise-updates/icehouse',
-    'icehouse/proposed': 'precise-proposed/icehouse',
-    'precise-icehouse/proposed': 'precise-proposed/icehouse',
-    'precise-proposed/icehouse': 'precise-proposed/icehouse',
-    # Juno
-    'juno': 'trusty-updates/juno',
-    'trusty-juno': 'trusty-updates/juno',
-    'trusty-juno/updates': 'trusty-updates/juno',
-    'trusty-updates/juno': 'trusty-updates/juno',
-    'juno/proposed': 'trusty-proposed/juno',
-    'trusty-juno/proposed': 'trusty-proposed/juno',
-    'trusty-proposed/juno': 'trusty-proposed/juno',
-    # Kilo
-    'kilo': 'trusty-updates/kilo',
-    'trusty-kilo': 'trusty-updates/kilo',
-    'trusty-kilo/updates': 'trusty-updates/kilo',
-    'trusty-updates/kilo': 'trusty-updates/kilo',
-    'kilo/proposed': 'trusty-proposed/kilo',
-    'trusty-kilo/proposed': 'trusty-proposed/kilo',
-    'trusty-proposed/kilo': 'trusty-proposed/kilo',
-    # Liberty
-    'liberty': 'trusty-updates/liberty',
-    'trusty-liberty': 'trusty-updates/liberty',
-    'trusty-liberty/updates': 'trusty-updates/liberty',
-    'trusty-updates/liberty': 'trusty-updates/liberty',
-    'liberty/proposed': 'trusty-proposed/liberty',
-    'trusty-liberty/proposed': 'trusty-proposed/liberty',
-    'trusty-proposed/liberty': 'trusty-proposed/liberty',
-}
-
 # The order of this list is very important. Handlers should be listed in from
 # least- to most-specific URL matching.
 FETCH_HANDLERS = (
@@ -108,10 +35,6 @@ FETCH_HANDLERS = (
     'charmhelpers.fetch.giturl.GitUrlFetchHandler',
 )
 
-APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
-APT_NO_LOCK_RETRY_DELAY = 10  # Wait 10 seconds between apt lock checks.
-APT_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
-
 
 class SourceConfigError(Exception):
     pass
@@ -149,180 +72,38 @@ class BaseFetchHandler(object):
         return urlunparse(parts)
 
 
-def filter_installed_packages(packages):
+__platform__ = get_platform()
-    """Returns a list of packages that require installation"""
+module = "charmhelpers.fetch.%s" % __platform__
-    cache = apt_cache()
+fetch = importlib.import_module(module)
-    _pkgs = []
-    for package in packages:
-        try:
-            p = cache[package]
-            p.current_ver or _pkgs.append(package)
-        except KeyError:
-            log('Package {} has no installation candidate.'.format(package),
-                level='WARNING')
-            _pkgs.append(package)
-    return _pkgs
 
+filter_installed_packages = fetch.filter_installed_packages
+install = fetch.install
+upgrade = fetch.upgrade
+update = fetch.update
+purge = fetch.purge
+add_source = fetch.add_source
 
-def apt_cache(in_memory=True):
+if __platform__ == "ubuntu":
-    """Build and return an apt cache"""
+    apt_cache = fetch.apt_cache
-    from apt import apt_pkg
+    apt_install = fetch.install
-    apt_pkg.init()
+    apt_update = fetch.update
-    if in_memory:
+    apt_upgrade = fetch.upgrade
-        apt_pkg.config.set("Dir::Cache::pkgcache", "")
+    apt_purge = fetch.purge
-        apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
+    apt_mark = fetch.apt_mark
-    return apt_pkg.Cache()
+    apt_hold = fetch.apt_hold
-
+    apt_unhold = fetch.apt_unhold
-
+    get_upstream_version = fetch.get_upstream_version
-def apt_install(packages, options=None, fatal=False):
+elif __platform__ == "centos":
-    """Install one or more packages"""
+    yum_search = fetch.yum_search
-    if options is None:
-        options = ['--option=Dpkg::Options::=--force-confold']
-
-    cmd = ['apt-get', '--assume-yes']
-    cmd.extend(options)
-    cmd.append('install')
-    if isinstance(packages, six.string_types):
-        cmd.append(packages)
-    else:
-        cmd.extend(packages)
-    log("Installing {} with options: {}".format(packages,
-                                                options))
-    _run_apt_command(cmd, fatal)
-
-
-def apt_upgrade(options=None, fatal=False, dist=False):
-    """Upgrade all packages"""
-    if options is None:
-        options = ['--option=Dpkg::Options::=--force-confold']
-
-    cmd = ['apt-get', '--assume-yes']
-    cmd.extend(options)
-    if dist:
-        cmd.append('dist-upgrade')
-    else:
-        cmd.append('upgrade')
-    log("Upgrading with options: {}".format(options))
-    _run_apt_command(cmd, fatal)
-
-
-def apt_update(fatal=False):
-    """Update local apt cache"""
-    cmd = ['apt-get', 'update']
-    _run_apt_command(cmd, fatal)
-
-
-def apt_purge(packages, fatal=False):
-    """Purge one or more packages"""
-    cmd = ['apt-get', '--assume-yes', 'purge']
-    if isinstance(packages, six.string_types):
-        cmd.append(packages)
-    else:
-        cmd.extend(packages)
-    log("Purging {}".format(packages))
-    _run_apt_command(cmd, fatal)
-
-
-def apt_mark(packages, mark, fatal=False):
-    """Flag one or more packages using apt-mark"""
-    cmd = ['apt-mark', mark]
-    if isinstance(packages, six.string_types):
-        cmd.append(packages)
-    else:
-        cmd.extend(packages)
-    log("Holding {}".format(packages))
-
-    if fatal:
-        subprocess.check_call(cmd, universal_newlines=True)
-    else:
-        subprocess.call(cmd, universal_newlines=True)
-
-
-def apt_hold(packages, fatal=False):
-    return apt_mark(packages, 'hold', fatal=fatal)
-
-
-def apt_unhold(packages, fatal=False):
-    return apt_mark(packages, 'unhold', fatal=fatal)
-
-
-def add_source(source, key=None):
-    """Add a package source to this system.
-
-    @param source: a URL or sources.list entry, as supported by
-    add-apt-repository(1). Examples::
-
-        ppa:charmers/example
-        deb https://stub:key@private.example.com/ubuntu trusty main
-
-    In addition:
-        'proposed:' may be used to enable the standard 'proposed'
-        pocket for the release.
-        'cloud:' may be used to activate official cloud archive pockets,
-        such as 'cloud:icehouse'
-        'distro' may be used as a noop
-
-    @param key: A key to be added to the system's APT keyring and used
-    to verify the signatures on packages. Ideally, this should be an
-    ASCII format GPG public key including the block headers. A GPG key
-    id may also be used, but be aware that only insecure protocols are
-    available to retrieve the actual public key from a public keyserver
-    placing your Juju environment at risk. ppa and cloud archive keys
-    are securely added automtically, so sould not be provided.
-    """
-    if source is None:
-        log('Source is not present. Skipping')
-        return
-
-    if (source.startswith('ppa:') or
-        source.startswith('http') or
-        source.startswith('deb ') or
-            source.startswith('cloud-archive:')):
-        subprocess.check_call(['add-apt-repository', '--yes', source])
-    elif source.startswith('cloud:'):
-        apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
-                    fatal=True)
-        pocket = source.split(':')[-1]
-        if pocket not in CLOUD_ARCHIVE_POCKETS:
-            raise SourceConfigError(
-                'Unsupported cloud: source option %s' %
-                pocket)
-        actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
-        with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
-            apt.write(CLOUD_ARCHIVE.format(actual_pocket))
-    elif source == 'proposed':
-        release = lsb_release()['DISTRIB_CODENAME']
-        with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
-            apt.write(PROPOSED_POCKET.format(release))
-    elif source == 'distro':
-        pass
-    else:
-        log("Unknown source: {!r}".format(source))
-
-    if key:
-        if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
-            with NamedTemporaryFile('w+') as key_file:
-                key_file.write(key)
-                key_file.flush()
-                key_file.seek(0)
-                subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
-        else:
-            # Note that hkp: is in no way a secure protocol. Using a
-            # GPG key id is pointless from a security POV unless you
-            # absolutely trust your network and DNS.
-            subprocess.check_call(['apt-key', 'adv', '--keyserver',
-                                   'hkp://keyserver.ubuntu.com:80', '--recv',
-                                   key])
 
 
 def configure_sources(update=False,
                       sources_var='install_sources',
                       keys_var='install_keys'):
-    """
+    """Configure multiple sources from charm configuration.
-    Configure multiple sources from charm configuration.
 
     The lists are encoded as yaml fragments in the configuration.
-    The frament needs to be included as a string. Sources and their
+    The fragment needs to be included as a string. Sources and their
     corresponding keys are of the types supported by add_source().
 
     Example config:
@@ -354,12 +135,11 @@ def configure_sources(update=False,
         for source, key in zip(sources, keys):
             add_source(source, key)
     if update:
-        apt_update(fatal=True)
+        fetch.update(fatal=True)
 
 
 def install_remote(source, *args, **kwargs):
-    """
+    """Install a file tree from a remote source.
-    Install a file tree from a remote source
 
     The specified source should be a url of the form:
         scheme://[host]/path[#[option=value][&...]]
@@ -382,19 +162,17 @@ def install_remote(source, *args, **kwargs):
     # We ONLY check for True here because can_handle may return a string
     # explaining why it can't handle a given source.
     handlers = [h for h in plugins() if h.can_handle(source) is True]
-    installed_to = None
     for handler in handlers:
         try:
-            installed_to = handler.install(source, *args, **kwargs)
+            return handler.install(source, *args, **kwargs)
         except UnhandledSource as e:
             log('Install source attempt unsuccessful: {}'.format(e),
                 level='WARNING')
-    if not installed_to:
+    raise UnhandledSource("No handler found for source {}".format(source))
-        raise UnhandledSource("No handler found for source {}".format(source))
-    return installed_to
 
 
 def install_from_config(config_var_name):
+    """Install a file from config."""
     charm_config = config()
     source = charm_config[config_var_name]
     return install_remote(source)
@@ -411,46 +189,9 @@ def plugins(fetch_handlers=None):
                 importlib.import_module(package),
                 classname)
             plugin_list.append(handler_class())
-        except (ImportError, AttributeError):
+        except NotImplementedError:
             # Skip missing plugins so that they can be ommitted from
             # installation if desired
             log("FetchHandler {} not found, skipping plugin".format(
                 handler_name))
     return plugin_list
-
-
-def _run_apt_command(cmd, fatal=False):
-    """
-    Run an APT command, checking output and retrying if the fatal flag is set
-    to True.
-
-    :param: cmd: str: The apt command to run.
-    :param: fatal: bool: Whether the command's output should be checked and
-        retried.
-    """
-    env = os.environ.copy()
-
-    if 'DEBIAN_FRONTEND' not in env:
-        env['DEBIAN_FRONTEND'] = 'noninteractive'
-
-    if fatal:
-        retry_count = 0
-        result = None
-
-        # If the command is considered "fatal", we need to retry if the apt
-        # lock was not acquired.
-
-        while result is None or result == APT_NO_LOCK:
-            try:
-                result = subprocess.check_call(cmd, env=env)
-            except subprocess.CalledProcessError as e:
-                retry_count = retry_count + 1
-                if retry_count > APT_NO_LOCK_RETRY_COUNT:
-                    raise
-                result = e.returncode
-                log("Couldn't acquire DPKG lock. Will retry in {} seconds."
-                    "".format(APT_NO_LOCK_RETRY_DELAY))
-                time.sleep(APT_NO_LOCK_RETRY_DELAY)
-
-    else:
-        subprocess.call(cmd, env=env)

charmhelpers/fetch/archiveurl.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import hashlib
@@ -108,7 +106,7 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
                 install_opener(opener)
         response = urlopen(source)
         try:
-            with open(dest, 'w') as dest_file:
+            with open(dest, 'wb') as dest_file:
                 dest_file.write(response.read())
         except Exception as e:
             if os.path.isfile(dest):

charmhelpers/fetch/bzrurl.py

@@ -1,78 +1,76 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from subprocess import check_call
 from charmhelpers.fetch import (
     BaseFetchHandler,
-    UnhandledSource
+    UnhandledSource,
+    filter_installed_packages,
+    install,
 )
 from charmhelpers.core.host import mkdir
 
-import six
-if six.PY3:
-    raise ImportError('bzrlib does not support Python3')
 
-try:
+if filter_installed_packages(['bzr']) != []:
-    from bzrlib.branch import Branch
+    install(['bzr'])
-    from bzrlib import bzrdir, workingtree, errors
+    if filter_installed_packages(['bzr']) != []:
-except ImportError:
+        raise NotImplementedError('Unable to install bzr')
-    from charmhelpers.fetch import apt_install
-    apt_install("python-bzrlib")
-    from bzrlib.branch import Branch
-    from bzrlib import bzrdir, workingtree, errors
 
 
 class BzrUrlFetchHandler(BaseFetchHandler):
-    """Handler for bazaar branches via generic and lp URLs"""
+    """Handler for bazaar branches via generic and lp URLs."""
+
     def can_handle(self, source):
         url_parts = self.parse_url(source)
-        if url_parts.scheme not in ('bzr+ssh', 'lp'):
+        if url_parts.scheme not in ('bzr+ssh', 'lp', ''):
             return False
+        elif not url_parts.scheme:
+            return os.path.exists(os.path.join(source, '.bzr'))
         else:
             return True
 
-    def branch(self, source, dest):
+    def branch(self, source, dest, revno=None):
-        url_parts = self.parse_url(source)
-        # If we use lp:branchname scheme we need to load plugins
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
-        if url_parts.scheme == "lp":
+        cmd_opts = []
-            from bzrlib.plugin import load_plugins
+        if revno:
-            load_plugins()
+            cmd_opts += ['-r', str(revno)]
-        try:
+        if os.path.exists(dest):
-            local_branch = bzrdir.BzrDir.create_branch_convenience(dest)
+            cmd = ['bzr', 'pull']
-        except errors.AlreadyControlDirError:
+            cmd += cmd_opts
-            local_branch = Branch.open(dest)
+            cmd += ['--overwrite', '-d', dest, source]
-        try:
+        else:
-            remote_branch = Branch.open(source)
+            cmd = ['bzr', 'branch']
-            remote_branch.push(local_branch)
+            cmd += cmd_opts
-            tree = workingtree.WorkingTree.open(dest)
+            cmd += [source, dest]
-            tree.update()
+        check_call(cmd)
-        except Exception as e:
-            raise e
 
-    def install(self, source):
+    def install(self, source, dest=None, revno=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
-        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+        if dest:
-                                branch_name)
+            dest_dir = os.path.join(dest, branch_name)
-        if not os.path.exists(dest_dir):
+        else:
-            mkdir(dest_dir, perms=0o755)
+            dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+                                    branch_name)
+
+        if dest and not os.path.exists(dest):
+            mkdir(dest, perms=0o755)
+
         try:
-            self.branch(source, dest_dir)
+            self.branch(source, dest_dir, revno)
         except OSError as e:
             raise UnhandledSource(e.strerror)
         return dest_dir

charmhelpers/fetch/centos.py

@@ -0,0 +1,171 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import os
+import time
+import six
+import yum
+
+from tempfile import NamedTemporaryFile
+from charmhelpers.core.hookenv import log
+
+YUM_NO_LOCK = 1  # The return code for "couldn't acquire lock" in YUM.
+YUM_NO_LOCK_RETRY_DELAY = 10  # Wait 10 seconds between apt lock checks.
+YUM_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+
+
+def filter_installed_packages(packages):
+    """Return a list of packages that require installation."""
+    yb = yum.YumBase()
+    package_list = yb.doPackageLists()
+    temp_cache = {p.base_package_name: 1 for p in package_list['installed']}
+
+    _pkgs = [p for p in packages if not temp_cache.get(p, False)]
+    return _pkgs
+
+
+def install(packages, options=None, fatal=False):
+    """Install one or more packages."""
+    cmd = ['yum', '--assumeyes']
+    if options is not None:
+        cmd.extend(options)
+    cmd.append('install')
+    if isinstance(packages, six.string_types):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Installing {} with options: {}".format(packages,
+                                                options))
+    _run_yum_command(cmd, fatal)
+
+
+def upgrade(options=None, fatal=False, dist=False):
+    """Upgrade all packages."""
+    cmd = ['yum', '--assumeyes']
+    if options is not None:
+        cmd.extend(options)
+    cmd.append('upgrade')
+    log("Upgrading with options: {}".format(options))
+    _run_yum_command(cmd, fatal)
+
+
+def update(fatal=False):
+    """Update local yum cache."""
+    cmd = ['yum', '--assumeyes', 'update']
+    log("Update with fatal: {}".format(fatal))
+    _run_yum_command(cmd, fatal)
+
+
+def purge(packages, fatal=False):
+    """Purge one or more packages."""
+    cmd = ['yum', '--assumeyes', 'remove']
+    if isinstance(packages, six.string_types):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Purging {}".format(packages))
+    _run_yum_command(cmd, fatal)
+
+
+def yum_search(packages):
+    """Search for a package."""
+    output = {}
+    cmd = ['yum', 'search']
+    if isinstance(packages, six.string_types):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Searching for {}".format(packages))
+    result = subprocess.check_output(cmd)
+    for package in list(packages):
+        output[package] = package in result
+    return output
+
+
+def add_source(source, key=None):
+    """Add a package source to this system.
+
+    @param source: a URL with a rpm package
+
+    @param key: A key to be added to the system's keyring and used
+    to verify the signatures on packages. Ideally, this should be an
+    ASCII format GPG public key including the block headers. A GPG key
+    id may also be used, but be aware that only insecure protocols are
+    available to retrieve the actual public key from a public keyserver
+    placing your Juju environment at risk.
+    """
+    if source is None:
+        log('Source is not present. Skipping')
+        return
+
+    if source.startswith('http'):
+        directory = '/etc/yum.repos.d/'
+        for filename in os.listdir(directory):
+            with open(directory + filename, 'r') as rpm_file:
+                if source in rpm_file.read():
+                    break
+        else:
+            log("Add source: {!r}".format(source))
+            # write in the charms.repo
+            with open(directory + 'Charms.repo', 'a') as rpm_file:
+                rpm_file.write('[%s]\n' % source[7:].replace('/', '_'))
+                rpm_file.write('name=%s\n' % source[7:])
+                rpm_file.write('baseurl=%s\n\n' % source)
+    else:
+        log("Unknown source: {!r}".format(source))
+
+    if key:
+        if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
+            with NamedTemporaryFile('w+') as key_file:
+                key_file.write(key)
+                key_file.flush()
+                key_file.seek(0)
+            subprocess.check_call(['rpm', '--import', key_file])
+        else:
+            subprocess.check_call(['rpm', '--import', key])
+
+
+def _run_yum_command(cmd, fatal=False):
+    """Run an YUM command.
+
+    Checks the output and retry if the fatal flag is set to True.
+
+    :param: cmd: str: The yum command to run.
+    :param: fatal: bool: Whether the command's output should be checked and
+        retried.
+    """
+    env = os.environ.copy()
+
+    if fatal:
+        retry_count = 0
+        result = None
+
+        # If the command is considered "fatal", we need to retry if the yum
+        # lock was not acquired.
+
+        while result is None or result == YUM_NO_LOCK:
+            try:
+                result = subprocess.check_call(cmd, env=env)
+            except subprocess.CalledProcessError as e:
+                retry_count = retry_count + 1
+                if retry_count > YUM_NO_LOCK_RETRY_COUNT:
+                    raise
+                result = e.returncode
+                log("Couldn't acquire YUM lock. Will retry in {} seconds."
+                    "".format(YUM_NO_LOCK_RETRY_DELAY))
+                time.sleep(YUM_NO_LOCK_RETRY_DELAY)
+
+    else:
+        subprocess.call(cmd, env=env)

charmhelpers/fetch/giturl.py

@@ -1,58 +1,56 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from subprocess import check_call, CalledProcessError
 from charmhelpers.fetch import (
     BaseFetchHandler,
-    UnhandledSource
+    UnhandledSource,
+    filter_installed_packages,
+    install,
 )
-from charmhelpers.core.host import mkdir
 
-import six
+if filter_installed_packages(['git']) != []:
-if six.PY3:
+    install(['git'])
-    raise ImportError('GitPython does not support Python 3')
+    if filter_installed_packages(['git']) != []:
-
+        raise NotImplementedError('Unable to install git')
-try:
-    from git import Repo
-except ImportError:
-    from charmhelpers.fetch import apt_install
-    apt_install("python-git")
-    from git import Repo
-
-from git.exc import GitCommandError  # noqa E402
 
 
 class GitUrlFetchHandler(BaseFetchHandler):
-    """Handler for git branches via generic and github URLs"""
+    """Handler for git branches via generic and github URLs."""
+
     def can_handle(self, source):
         url_parts = self.parse_url(source)
         # TODO (mattyw) no support for ssh git@ yet
-        if url_parts.scheme not in ('http', 'https', 'git'):
+        if url_parts.scheme not in ('http', 'https', 'git', ''):
             return False
+        elif not url_parts.scheme:
+            return os.path.exists(os.path.join(source, '.git'))
         else:
             return True
 
-    def clone(self, source, dest, branch, depth=None):
+    def clone(self, source, dest, branch="master", depth=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 
-        if depth:
+        if os.path.exists(dest):
-            Repo.clone_from(source, dest, branch=branch, depth=depth)
+            cmd = ['git', '-C', dest, 'pull', source, branch]
         else:
-            Repo.clone_from(source, dest, branch=branch)
+            cmd = ['git', 'clone', source, dest, '--branch', branch]
+            if depth:
+                cmd.extend(['--depth', depth])
+        check_call(cmd)
 
     def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)
@@ -62,11 +60,9 @@ class GitUrlFetchHandler(BaseFetchHandler):
         else:
             dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
                                     branch_name)
-        if not os.path.exists(dest_dir):
-            mkdir(dest_dir, perms=0o755)
         try:
             self.clone(source, dest_dir, branch, depth)
-        except GitCommandError as e:
+        except CalledProcessError as e:
             raise UnhandledSource(e)
         except OSError as e:
             raise UnhandledSource(e.strerror)

charmhelpers/fetch/snap.py

@@ -0,0 +1,122 @@
+# Copyright 2014-2017 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Charm helpers snap for classic charms.
+
+If writing reactive charms, use the snap layer:
+https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
+"""
+import subprocess
+from os import environ
+from time import sleep
+from charmhelpers.core.hookenv import log
+
+__author__ = 'Joseph Borg <joseph.borg@canonical.com>'
+
+SNAP_NO_LOCK = 1  # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
+SNAP_NO_LOCK_RETRY_DELAY = 10  # Wait X seconds between Snap lock checks.
+SNAP_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+
+
+class CouldNotAcquireLockException(Exception):
+    pass
+
+
+def _snap_exec(commands):
+    """
+    Execute snap commands.
+
+    :param commands: List commands
+    :return: Integer exit code
+    """
+    assert type(commands) == list
+
+    retry_count = 0
+    return_code = None
+
+    while return_code is None or return_code == SNAP_NO_LOCK:
+        try:
+            return_code = subprocess.check_call(['snap'] + commands, env=environ)
+        except subprocess.CalledProcessError as e:
+            retry_count += + 1
+            if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
+                raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
+            return_code = e.returncode
+            log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
+            sleep(SNAP_NO_LOCK_RETRY_DELAY)
+
+    return return_code
+
+
+def snap_install(packages, *flags):
+    """
+    Install a snap package.
+
+    :param packages: String or List String package name
+    :param flags: List String flags to pass to install command
+    :return: Integer return code from snap
+    """
+    if type(packages) is not list:
+        packages = [packages]
+
+    flags = list(flags)
+
+    message = 'Installing snap(s) "%s"' % ', '.join(packages)
+    if flags:
+        message += ' with option(s) "%s"' % ', '.join(flags)
+
+    log(message, level='INFO')
+    return _snap_exec(['install'] + flags + packages)
+
+
+def snap_remove(packages, *flags):
+    """
+    Remove a snap package.
+
+    :param packages: String or List String package name
+    :param flags: List String flags to pass to remove command
+    :return: Integer return code from snap
+    """
+    if type(packages) is not list:
+        packages = [packages]
+
+    flags = list(flags)
+
+    message = 'Removing snap(s) "%s"' % ', '.join(packages)
+    if flags:
+        message += ' with options "%s"' % ', '.join(flags)
+
+    log(message, level='INFO')
+    return _snap_exec(['remove'] + flags + packages)
+
+
+def snap_refresh(packages, *flags):
+    """
+    Refresh / Update snap package.
+
+    :param packages: String or List String package name
+    :param flags: List String flags to pass to refresh command
+    :return: Integer return code from snap
+    """
+    if type(packages) is not list:
+        packages = [packages]
+
+    flags = list(flags)
+
+    message = 'Refreshing snap(s) "%s"' % ', '.join(packages)
+    if flags:
+        message += ' with options "%s"' % ', '.join(flags)
+
+    log(message, level='INFO')
+    return _snap_exec(['refresh'] + flags + packages)

charmhelpers/fetch/ubuntu.py

@@ -0,0 +1,364 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import six
+import time
+import subprocess
+
+from tempfile import NamedTemporaryFile
+from charmhelpers.core.host import (
+    lsb_release
+)
+from charmhelpers.core.hookenv import log
+from charmhelpers.fetch import SourceConfigError
+
+CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+"""
+
+PROPOSED_POCKET = """# Proposed
+deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
+"""
+
+CLOUD_ARCHIVE_POCKETS = {
+    # Folsom
+    'folsom': 'precise-updates/folsom',
+    'precise-folsom': 'precise-updates/folsom',
+    'precise-folsom/updates': 'precise-updates/folsom',
+    'precise-updates/folsom': 'precise-updates/folsom',
+    'folsom/proposed': 'precise-proposed/folsom',
+    'precise-folsom/proposed': 'precise-proposed/folsom',
+    'precise-proposed/folsom': 'precise-proposed/folsom',
+    # Grizzly
+    'grizzly': 'precise-updates/grizzly',
+    'precise-grizzly': 'precise-updates/grizzly',
+    'precise-grizzly/updates': 'precise-updates/grizzly',
+    'precise-updates/grizzly': 'precise-updates/grizzly',
+    'grizzly/proposed': 'precise-proposed/grizzly',
+    'precise-grizzly/proposed': 'precise-proposed/grizzly',
+    'precise-proposed/grizzly': 'precise-proposed/grizzly',
+    # Havana
+    'havana': 'precise-updates/havana',
+    'precise-havana': 'precise-updates/havana',
+    'precise-havana/updates': 'precise-updates/havana',
+    'precise-updates/havana': 'precise-updates/havana',
+    'havana/proposed': 'precise-proposed/havana',
+    'precise-havana/proposed': 'precise-proposed/havana',
+    'precise-proposed/havana': 'precise-proposed/havana',
+    # Icehouse
+    'icehouse': 'precise-updates/icehouse',
+    'precise-icehouse': 'precise-updates/icehouse',
+    'precise-icehouse/updates': 'precise-updates/icehouse',
+    'precise-updates/icehouse': 'precise-updates/icehouse',
+    'icehouse/proposed': 'precise-proposed/icehouse',
+    'precise-icehouse/proposed': 'precise-proposed/icehouse',
+    'precise-proposed/icehouse': 'precise-proposed/icehouse',
+    # Juno
+    'juno': 'trusty-updates/juno',
+    'trusty-juno': 'trusty-updates/juno',
+    'trusty-juno/updates': 'trusty-updates/juno',
+    'trusty-updates/juno': 'trusty-updates/juno',
+    'juno/proposed': 'trusty-proposed/juno',
+    'trusty-juno/proposed': 'trusty-proposed/juno',
+    'trusty-proposed/juno': 'trusty-proposed/juno',
+    # Kilo
+    'kilo': 'trusty-updates/kilo',
+    'trusty-kilo': 'trusty-updates/kilo',
+    'trusty-kilo/updates': 'trusty-updates/kilo',
+    'trusty-updates/kilo': 'trusty-updates/kilo',
+    'kilo/proposed': 'trusty-proposed/kilo',
+    'trusty-kilo/proposed': 'trusty-proposed/kilo',
+    'trusty-proposed/kilo': 'trusty-proposed/kilo',
+    # Liberty
+    'liberty': 'trusty-updates/liberty',
+    'trusty-liberty': 'trusty-updates/liberty',
+    'trusty-liberty/updates': 'trusty-updates/liberty',
+    'trusty-updates/liberty': 'trusty-updates/liberty',
+    'liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-proposed/liberty': 'trusty-proposed/liberty',
+    # Mitaka
+    'mitaka': 'trusty-updates/mitaka',
+    'trusty-mitaka': 'trusty-updates/mitaka',
+    'trusty-mitaka/updates': 'trusty-updates/mitaka',
+    'trusty-updates/mitaka': 'trusty-updates/mitaka',
+    'mitaka/proposed': 'trusty-proposed/mitaka',
+    'trusty-mitaka/proposed': 'trusty-proposed/mitaka',
+    'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
+    # Newton
+    'newton': 'xenial-updates/newton',
+    'xenial-newton': 'xenial-updates/newton',
+    'xenial-newton/updates': 'xenial-updates/newton',
+    'xenial-updates/newton': 'xenial-updates/newton',
+    'newton/proposed': 'xenial-proposed/newton',
+    'xenial-newton/proposed': 'xenial-proposed/newton',
+    'xenial-proposed/newton': 'xenial-proposed/newton',
+    # Ocata
+    'ocata': 'xenial-updates/ocata',
+    'xenial-ocata': 'xenial-updates/ocata',
+    'xenial-ocata/updates': 'xenial-updates/ocata',
+    'xenial-updates/ocata': 'xenial-updates/ocata',
+    'ocata/proposed': 'xenial-proposed/ocata',
+    'xenial-ocata/proposed': 'xenial-proposed/ocata',
+    'xenial-ocata/newton': 'xenial-proposed/ocata',
+}
+
+APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
+CMD_RETRY_DELAY = 10  # Wait 10 seconds between command retries.
+CMD_RETRY_COUNT = 30  # Retry a failing fatal command X times.
+
+
+def filter_installed_packages(packages):
+    """Return a list of packages that require installation."""
+    cache = apt_cache()
+    _pkgs = []
+    for package in packages:
+        try:
+            p = cache[package]
+            p.current_ver or _pkgs.append(package)
+        except KeyError:
+            log('Package {} has no installation candidate.'.format(package),
+                level='WARNING')
+            _pkgs.append(package)
+    return _pkgs
+
+
+def apt_cache(in_memory=True, progress=None):
+    """Build and return an apt cache."""
+    from apt import apt_pkg
+    apt_pkg.init()
+    if in_memory:
+        apt_pkg.config.set("Dir::Cache::pkgcache", "")
+        apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
+    return apt_pkg.Cache(progress)
+
+
+def install(packages, options=None, fatal=False):
+    """Install one or more packages."""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    cmd.append('install')
+    if isinstance(packages, six.string_types):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Installing {} with options: {}".format(packages,
+                                                options))
+    _run_apt_command(cmd, fatal)
+
+
+def upgrade(options=None, fatal=False, dist=False):
+    """Upgrade all packages."""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    if dist:
+        cmd.append('dist-upgrade')
+    else:
+        cmd.append('upgrade')
+    log("Upgrading with options: {}".format(options))
+    _run_apt_command(cmd, fatal)
+
+
+def update(fatal=False):
+    """Update local apt cache."""
+    cmd = ['apt-get', 'update']
+    _run_apt_command(cmd, fatal)
+
+
+def purge(packages, fatal=False):
+    """Purge one or more packages."""
+    cmd = ['apt-get', '--assume-yes', 'purge']
+    if isinstance(packages, six.string_types):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Purging {}".format(packages))
+    _run_apt_command(cmd, fatal)
+
+
+def apt_mark(packages, mark, fatal=False):
+    """Flag one or more packages using apt-mark."""
+    log("Marking {} as {}".format(packages, mark))
+    cmd = ['apt-mark', mark]
+    if isinstance(packages, six.string_types):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+
+    if fatal:
+        subprocess.check_call(cmd, universal_newlines=True)
+    else:
+        subprocess.call(cmd, universal_newlines=True)
+
+
+def apt_hold(packages, fatal=False):
+    return apt_mark(packages, 'hold', fatal=fatal)
+
+
+def apt_unhold(packages, fatal=False):
+    return apt_mark(packages, 'unhold', fatal=fatal)
+
+
+def add_source(source, key=None):
+    """Add a package source to this system.
+
+    @param source: a URL or sources.list entry, as supported by
+    add-apt-repository(1). Examples::
+
+        ppa:charmers/example
+        deb https://stub:key@private.example.com/ubuntu trusty main
+
+    In addition:
+        'proposed:' may be used to enable the standard 'proposed'
+        pocket for the release.
+        'cloud:' may be used to activate official cloud archive pockets,
+        such as 'cloud:icehouse'
+        'distro' may be used as a noop
+
+    @param key: A key to be added to the system's APT keyring and used
+    to verify the signatures on packages. Ideally, this should be an
+    ASCII format GPG public key including the block headers. A GPG key
+    id may also be used, but be aware that only insecure protocols are
+    available to retrieve the actual public key from a public keyserver
+    placing your Juju environment at risk. ppa and cloud archive keys
+    are securely added automtically, so sould not be provided.
+    """
+    if source is None:
+        log('Source is not present. Skipping')
+        return
+
+    if (source.startswith('ppa:') or
+        source.startswith('http') or
+        source.startswith('deb ') or
+            source.startswith('cloud-archive:')):
+        cmd = ['add-apt-repository', '--yes', source]
+        _run_with_retries(cmd)
+    elif source.startswith('cloud:'):
+        install(filter_installed_packages(['ubuntu-cloud-keyring']),
+                fatal=True)
+        pocket = source.split(':')[-1]
+        if pocket not in CLOUD_ARCHIVE_POCKETS:
+            raise SourceConfigError(
+                'Unsupported cloud: source option %s' %
+                pocket)
+        actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
+        with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+            apt.write(CLOUD_ARCHIVE.format(actual_pocket))
+    elif source == 'proposed':
+        release = lsb_release()['DISTRIB_CODENAME']
+        with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
+            apt.write(PROPOSED_POCKET.format(release))
+    elif source == 'distro':
+        pass
+    else:
+        log("Unknown source: {!r}".format(source))
+
+    if key:
+        if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
+            with NamedTemporaryFile('w+') as key_file:
+                key_file.write(key)
+                key_file.flush()
+                key_file.seek(0)
+                subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
+        else:
+            # Note that hkp: is in no way a secure protocol. Using a
+            # GPG key id is pointless from a security POV unless you
+            # absolutely trust your network and DNS.
+            subprocess.check_call(['apt-key', 'adv', '--keyserver',
+                                   'hkp://keyserver.ubuntu.com:80', '--recv',
+                                   key])
+
+
+def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
+                      retry_message="", cmd_env=None):
+    """Run a command and retry until success or max_retries is reached.
+
+    :param: cmd: str: The apt command to run.
+    :param: max_retries: int: The number of retries to attempt on a fatal
+        command. Defaults to CMD_RETRY_COUNT.
+    :param: retry_exitcodes: tuple: Optional additional exit codes to retry.
+        Defaults to retry on exit code 1.
+    :param: retry_message: str: Optional log prefix emitted during retries.
+    :param: cmd_env: dict: Environment variables to add to the command run.
+    """
+
+    env = os.environ.copy()
+    if cmd_env:
+        env.update(cmd_env)
+
+    if not retry_message:
+        retry_message = "Failed executing '{}'".format(" ".join(cmd))
+    retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY)
+
+    retry_count = 0
+    result = None
+
+    retry_results = (None,) + retry_exitcodes
+    while result in retry_results:
+        try:
+            result = subprocess.check_call(cmd, env=env)
+        except subprocess.CalledProcessError as e:
+            retry_count = retry_count + 1
+            if retry_count > max_retries:
+                raise
+            result = e.returncode
+            log(retry_message)
+            time.sleep(CMD_RETRY_DELAY)
+
+
+def _run_apt_command(cmd, fatal=False):
+    """Run an apt command with optional retries.
+
+    :param: fatal: bool: Whether the command's output should be checked and
+        retried.
+    """
+    # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment.
+    cmd_env = {
+        'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')}
+
+    if fatal:
+        _run_with_retries(
+            cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,),
+            retry_message="Couldn't acquire DPKG lock")
+    else:
+        env = os.environ.copy()
+        env.update(cmd_env)
+        subprocess.call(cmd, env=env)
+
+
+def get_upstream_version(package):
+    """Determine upstream version based on installed package
+
+    @returns None (if not installed) or the upstream version
+    """
+    import apt_pkg
+    cache = apt_cache()
+    try:
+        pkg = cache[package]
+    except:
+        # the package is unknown to the current apt cache.
+        return None
+
+    if not pkg.current_ver:
+        # package is known, but no version is currently installed.
+        return None
+
+    return apt_pkg.upstream_version(pkg.current_ver.ver_str)

charmhelpers/osplatform.py

@@ -0,0 +1,25 @@
+import platform
+
+
+def get_platform():
+    """Return the current OS platform.
+
+    For example: if current os platform is Ubuntu then a string "ubuntu"
+    will be returned (which is the name of the module).
+    This string is used to decide which platform module should be imported.
+    """
+    # linux_distribution is deprecated and will be removed in Python 3.7
+    # Warings *not* disabled, as we certainly need to fix this.
+    tuple_platform = platform.linux_distribution()
+    current_platform = tuple_platform[0]
+    if "Ubuntu" in current_platform:
+        return "ubuntu"
+    elif "CentOS" in current_platform:
+        return "centos"
+    elif "debian" in current_platform:
+        # Stock Python does not detect Ubuntu and instead returns debian.
+        # Or at least it does in some build environments like Travis CI
+        return "ubuntu"
+    else:
+        raise RuntimeError("This module is not supported on {}."
+                           .format(current_platform))

charmhelpers/payload/__init__.py

@@ -1,17 +1,15 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 "Tools for working with files injected into a charm just before deployment."

charmhelpers/payload/archive.py

@@ -1,18 +1,16 @@
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import tarfile

charmhelpers/payload/execd.py

@@ -2,19 +2,17 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
+#  http://www.apache.org/licenses/LICENSE-2.0
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
 #
-# charm-helpers is distributed in the hope that it will be useful,
+# Unless required by applicable law or agreed to in writing, software
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# distributed under the License is distributed on an "AS IS" BASIS,
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# GNU Lesser General Public License for more details.
+# See the License for the specific language governing permissions and
-#
+# limitations under the License.
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import sys
@@ -49,11 +47,12 @@ def execd_submodule_paths(command, execd_dir=None):
             yield path
 
 
-def execd_run(command, execd_dir=None, die_on_error=False, stderr=None):
+def execd_run(command, execd_dir=None, die_on_error=True, stderr=subprocess.STDOUT):
     """Run command for each module within execd_dir which defines it."""
     for submodule_path in execd_submodule_paths(command, execd_dir):
         try:
-            subprocess.check_call(submodule_path, shell=True, stderr=stderr)
+            subprocess.check_output(submodule_path, stderr=stderr,
+                                    universal_newlines=True)
         except subprocess.CalledProcessError as e:
             hookenv.log("Error ({}) running  {}. Output: {}".format(
                 e.returncode, e.cmd, e.output))

unit_tests/test_hooks.py

@@ -41,6 +41,7 @@ class TestConfigChanged(CharmTestCase):
         shutil.rmtree(self.tmpcrond)
         shutil.rmtree(self.sharedir)
 
+    @mock.patch.object(hooks, 'update_nrpe_config')
     @mock.patch('os.symlink')
     @mock.patch('hooks.charmhelpers.core.hookenv.config')
     @mock.patch('hooks.charmhelpers.core.hookenv.relations_of_type')
@@ -49,7 +50,8 @@ class TestConfigChanged(CharmTestCase):
     @mock.patch('hooks.charmhelpers.contrib.charmsupport.nrpe.config')
     @mock.patch('hooks.charmhelpers.contrib.charmsupport.nrpe.local_unit')
     def test_default_config(self, local_unit, nrpe_config, nag_host,
-                            relations_of_type, config, symlink):
+                            relations_of_type, config, symlink,
+                            update_nrpe_config):
         local_unit.return_value = 'juju/0'
         nag_host.return_value = "nagios_hostname"
         nrpe_config.return_value = self.test_config
@@ -74,7 +76,9 @@ class TestConfigChanged(CharmTestCase):
 
         mirror_list = yaml.safe_load(self.test_config['mirror_list'])
         self.assertEqual(mirrors['mirror_list'], mirror_list)
+        update_nrpe_config.assert_called()
 
+    @mock.patch.object(hooks, 'update_nrpe_config')
     @mock.patch('os.path.exists')
     @mock.patch('os.remove')
     @mock.patch('glob.glob')
@@ -85,7 +89,8 @@ class TestConfigChanged(CharmTestCase):
     @mock.patch('hooks.charmhelpers.contrib.charmsupport.nrpe.config')
     @mock.patch('hooks.charmhelpers.contrib.charmsupport.nrpe.local_unit')
     def test_uninstall_cron(self, local_unit, nrpe_config, nag_host,
-                            relations_of_type, config, glob, remove, exists):
+                            relations_of_type, config, glob, remove, exists,
+                            update_nrpe_config):
         local_unit.return_value = 'juju/0'
         nag_host.return_value = "nagios_hostname"
         nrpe_config.return_value = self.test_config
@@ -101,3 +106,4 @@ class TestConfigChanged(CharmTestCase):
         remove.assert_any_call(os.path.join('/etc/cron.daily/',
                                             hooks.CRON_JOB_FILENAME))
         remove.assert_any_call(hooks.CRON_POLL_FILEPATH)
+        update_nrpe_config.assert_called()