[corey.bryant,r=trivial] Sync charm-helpers to pick up Liberty support.

hooks/charmhelpers/cli/__init__.py

@@ -152,15 +152,11 @@ class CommandLine(object):
         arguments = self.argument_parser.parse_args()
         argspec = inspect.getargspec(arguments.func)
         vargs = []
-        kwargs = {}
         for arg in argspec.args:
             vargs.append(getattr(arguments, arg))
         if argspec.varargs:
             vargs.extend(getattr(arguments, argspec.varargs))
-        if argspec.keywords:
+        output = arguments.func(*vargs)
-            for kwarg in argspec.keywords.items():
-                kwargs[kwarg] = getattr(arguments, kwarg)
-        output = arguments.func(*vargs, **kwargs)
         if getattr(arguments.func, '_cli_test_command', False):
             self.exit_code = 0 if output else 1
             output = ''

hooks/charmhelpers/cli/commands.py

@@ -26,7 +26,7 @@ from . import CommandLine  # noqa
 """
 Import the sub-modules which have decorated subcommands to register with chlp.
 """
-import host  # noqa
+from . import host  # noqa
-import benchmark  # noqa
+from . import benchmark  # noqa
-import unitdata  # noqa
+from . import unitdata  # noqa
-from charmhelpers.core import hookenv  # noqa
+from . import hookenv  # noqa

hooks/charmhelpers/cli/hookenv.py

@@ -0,0 +1,23 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.core import hookenv
+
+
+cmdline.subcommand('relation-id')(hookenv.relation_id._wrapped)
+cmdline.subcommand('service-name')(hookenv.service_name)
+cmdline.subcommand('remote-service-name')(hookenv.remote_service_name._wrapped)

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+        base_charms = ['mysql', 'mongodb', 'nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
@@ -81,7 +81,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                       'ceph-osd', 'ceph-radosgw']
         # Most OpenStack subordinate charms do not expose an origin option
         # as that is controlled by the principle.
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services:

hooks/charmhelpers/contrib/openstack/utils.py

@@ -24,6 +24,7 @@ import subprocess
 import json
 import os
 import sys
+import re
 
 import six
 import yaml
@@ -69,7 +70,6 @@ CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
 DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
                    'restricted main multiverse universe')
 
-
 UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('oneiric', 'diablo'),
     ('precise', 'essex'),
@@ -118,6 +118,34 @@ SWIFT_CODENAMES = OrderedDict([
     ('2.3.0', 'liberty'),
 ])
 
+# >= Liberty version->codename mapping
+PACKAGE_CODENAMES = {
+    'nova-common': OrderedDict([
+        ('12.0.0', 'liberty'),
+    ]),
+    'neutron-common': OrderedDict([
+        ('7.0.0', 'liberty'),
+    ]),
+    'cinder-common': OrderedDict([
+        ('7.0.0', 'liberty'),
+    ]),
+    'keystone': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+    'horizon-common': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+    'ceilometer-common': OrderedDict([
+        ('5.0.0', 'liberty'),
+    ]),
+    'heat-common': OrderedDict([
+        ('5.0.0', 'liberty'),
+    ]),
+    'glance-common': OrderedDict([
+        ('11.0.0', 'liberty'),
+    ]),
+}
+
 DEFAULT_LOOPBACK_SIZE = '5G'
 
 
@@ -201,20 +229,29 @@ def get_os_codename_package(package, fatal=True):
         error_out(e)
 
     vers = apt.upstream_version(pkg.current_ver.ver_str)
+    match = re.match('^(\d)\.(\d)\.(\d)', vers)
+    if match:
+        vers = match.group(0)
 
-    try:
+    # >= Liberty independent project versions
-        if 'swift' in pkg.name:
+    if (package in PACKAGE_CODENAMES and
-            swift_vers = vers[:5]
+            vers in PACKAGE_CODENAMES[package]):
-            if swift_vers not in SWIFT_CODENAMES:
+        return PACKAGE_CODENAMES[package][vers]
-                # Deal with 1.10.0 upward
+    else:
-                swift_vers = vers[:6]
+        # < Liberty co-ordinated project versions
-            return SWIFT_CODENAMES[swift_vers]
+        try:
-        else:
+            if 'swift' in pkg.name:
-            vers = vers[:6]
+                swift_vers = vers[:5]
-            return OPENSTACK_CODENAMES[vers]
+                if swift_vers not in SWIFT_CODENAMES:
-    except KeyError:
+                    # Deal with 1.10.0 upward
-        e = 'Could not determine OpenStack codename for version %s' % vers
+                    swift_vers = vers[:6]
-        error_out(e)
+                return SWIFT_CODENAMES[swift_vers]
+            else:
+                vers = vers[:6]
+                return OPENSTACK_CODENAMES[vers]
+        except KeyError:
+            e = 'Could not determine OpenStack codename for version %s' % vers
+            error_out(e)
 
 
 def get_os_version_package(pkg, fatal=True):

hooks/charmhelpers/contrib/storage/linux/utils.py

@@ -43,9 +43,10 @@ def zap_disk(block_device):
 
     :param block_device: str: Full path of block device to clean.
     '''
+    # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b
     # sometimes sgdisk exits non-zero; this is OK, dd will clean up
-    call(['sgdisk', '--zap-all', '--mbrtogpt',
+    call(['sgdisk', '--zap-all', '--', block_device])
-          '--clear', block_device])
+    call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device])
     dev_end = check_output(['blockdev', '--getsz',
                             block_device]).decode('UTF-8')
     gpt_end = int(dev_end.split()[0]) - 100

hooks/charmhelpers/core/hookenv.py

@@ -34,23 +34,6 @@ import errno
 import tempfile
 from subprocess import CalledProcessError
 
-try:
-    from charmhelpers.cli import cmdline
-except ImportError as e:
-    # due to the anti-pattern of partially synching charmhelpers directly
-    # into charms, it's possible that charmhelpers.cli is not available;
-    # if that's the case, they don't really care about using the cli anyway,
-    # so mock it out
-    if str(e) == 'No module named cli':
-        class cmdline(object):
-            @classmethod
-            def subcommand(cls, *args, **kwargs):
-                def _wrap(func):
-                    return func
-                return _wrap
-    else:
-        raise
-
 import six
 if not six.PY3:
     from UserDict import UserDict
@@ -91,6 +74,7 @@ def cached(func):
         res = func(*args, **kwargs)
         cache[key] = res
         return res
+    wrapper._wrapped = func
     return wrapper
 
 
@@ -190,7 +174,6 @@ def relation_type():
     return os.environ.get('JUJU_RELATION', None)
 
 
-@cmdline.subcommand()
 @cached
 def relation_id(relation_name=None, service_or_unit=None):
     """The relation ID for the current or a specified relation"""
@@ -216,13 +199,11 @@ def remote_unit():
     return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
-@cmdline.subcommand()
 def service_name():
     """The name service group this unit belongs to"""
     return local_unit().split('/')[0]
 
 
-@cmdline.subcommand()
 @cached
 def remote_service_name(relid=None):
     """The remote service name for a given relation-id (or the current relation)"""

hooks/charmhelpers/core/host.py

@@ -72,7 +72,7 @@ def service_pause(service_name, init_dir=None):
     stopped = service_stop(service_name)
     # XXX: Support systemd too
     override_path = os.path.join(
-        init_dir, '{}.conf.override'.format(service_name))
+        init_dir, '{}.override'.format(service_name))
     with open(override_path, 'w') as fh:
         fh.write("manual\n")
     return stopped
@@ -86,7 +86,7 @@ def service_resume(service_name, init_dir=None):
     if init_dir is None:
         init_dir = "/etc/init"
     override_path = os.path.join(
-        init_dir, '{}.conf.override'.format(service_name))
+        init_dir, '{}.override'.format(service_name))
     if os.path.exists(override_path):
         os.unlink(override_path)
     started = service_start(service_name)
@@ -148,6 +148,16 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
     return user_info
 
 
+def user_exists(username):
+    """Check if a user exists"""
+    try:
+        pwd.getpwnam(username)
+        user_exists = True
+    except KeyError:
+        user_exists = False
+    return user_exists
+
+
 def add_group(group_name, system_group=False):
     """Add a group to the system"""
     try:
@@ -280,6 +290,17 @@ def mounts():
     return system_mounts
 
 
+def fstab_mount(mountpoint):
+    """Mount filesystem using fstab"""
+    cmd_args = ['mount', mountpoint]
+    try:
+        subprocess.check_output(cmd_args)
+    except subprocess.CalledProcessError as e:
+        log('Error unmounting {}\n{}'.format(mountpoint, e.output))
+        return False
+    return True
+
+
 def file_hash(path, hash_type='md5'):
     """
     Generate a hash checksum of the contents of 'path' or None if not found.

hooks/charmhelpers/core/services/helpers.py

@@ -16,7 +16,9 @@
 
 import os
 import yaml
+
 from charmhelpers.core import hookenv
+from charmhelpers.core import host
 from charmhelpers.core import templating
 
 from charmhelpers.core.services.base import ManagerCallback
@@ -240,27 +242,41 @@ class TemplateCallback(ManagerCallback):
 
     :param str source: The template source file, relative to
         `$CHARM_DIR/templates`
+
     :param str target: The target to write the rendered template to
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
-
+    :param partial on_change_action: functools partial to be executed when
+                                     rendered file changes
     """
     def __init__(self, source, target,
-                 owner='root', group='root', perms=0o444):
+                 owner='root', group='root', perms=0o444,
+                 on_change_action=None):
         self.source = source
         self.target = target
         self.owner = owner
         self.group = group
         self.perms = perms
+        self.on_change_action = on_change_action
 
     def __call__(self, manager, service_name, event_name):
+        pre_checksum = ''
+        if self.on_change_action and os.path.isfile(self.target):
+            pre_checksum = host.file_hash(self.target)
         service = manager.get_service(service_name)
         context = {}
         for ctx in service.get('required_data', []):
             context.update(ctx)
         templating.render(self.source, self.target, context,
                           self.owner, self.group, self.perms)
+        if self.on_change_action:
+            if pre_checksum == host.file_hash(self.target):
+                hookenv.log(
+                    'No change detected: {}'.format(self.target),
+                    hookenv.DEBUG)
+            else:
+                self.on_change_action()
 
 
 # Convenience aliases for templates

hooks/charmhelpers/fetch/__init__.py

@@ -90,6 +90,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'kilo/proposed': 'trusty-proposed/kilo',
     'trusty-kilo/proposed': 'trusty-proposed/kilo',
     'trusty-proposed/kilo': 'trusty-proposed/kilo',
+    # Liberty
+    'liberty': 'trusty-updates/liberty',
+    'trusty-liberty': 'trusty-updates/liberty',
+    'trusty-liberty/updates': 'trusty-updates/liberty',
+    'trusty-updates/liberty': 'trusty-updates/liberty',
+    'liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-proposed/liberty': 'trusty-proposed/liberty',
 }
 
 # The order of this list is very important. Handlers should be listed in from

tests/charmhelpers/contrib/amulet/utils.py

@@ -14,17 +14,23 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
-import amulet
-import ConfigParser
-import distro_info
 import io
+import json
 import logging
 import os
 import re
-import six
+import subprocess
 import sys
 import time
-import urlparse
+
+import amulet
+import distro_info
+import six
+from six.moves import configparser
+if six.PY3:
+    from urllib import parse as urlparse
+else:
+    import urlparse
 
 
 class AmuletUtils(object):
@@ -142,19 +148,23 @@ class AmuletUtils(object):
 
             for service_name in services_list:
                 if (self.ubuntu_releases.index(release) >= systemd_switch or
-                        service_name == "rabbitmq-server"):
+                        service_name in ['rabbitmq-server', 'apache2']):
-                    # init is systemd
+                    # init is systemd (or regular sysv)
                     cmd = 'sudo service {} status'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0
                 elif self.ubuntu_releases.index(release) < systemd_switch:
                     # init is upstart
                     cmd = 'sudo status {}'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0 and "start/running" in output
 
-                output, code = sentry_unit.run(cmd)
                 self.log.debug('{} `{}` returned '
                                '{}'.format(sentry_unit.info['unit_name'],
                                            cmd, code))
-                if code != 0:
+                if not service_running:
-                    return "command `{}` returned {}".format(cmd, str(code))
+                    return u"command `{}` returned {} {}".format(
+                        cmd, output, str(code))
         return None
 
     def _get_config(self, unit, filename):
@@ -164,7 +174,7 @@ class AmuletUtils(object):
         # NOTE(beisner):  by default, ConfigParser does not handle options
         # with no value, such as the flags used in the mysql my.cnf file.
         # https://bugs.python.org/issue7005
-        config = ConfigParser.ConfigParser(allow_no_value=True)
+        config = configparser.ConfigParser(allow_no_value=True)
         config.readfp(io.StringIO(file_contents))
         return config
 
@@ -450,15 +460,20 @@ class AmuletUtils(object):
                                         cmd, code, output))
         return None
 
-    def get_process_id_list(self, sentry_unit, process_name):
+    def get_process_id_list(self, sentry_unit, process_name,
+                            expect_success=True):
         """Get a list of process ID(s) from a single sentry juju unit
         for a single process name.
 
-        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param sentry_unit: Amulet sentry instance (juju unit)
         :param process_name: Process name
+        :param expect_success: If False, expect the PID to be missing,
+            raise if it is present.
         :returns: List of process IDs
         """
-        cmd = 'pidof {}'.format(process_name)
+        cmd = 'pidof -x {}'.format(process_name)
+        if not expect_success:
+            cmd += " || exit 0 && exit 1"
         output, code = sentry_unit.run(cmd)
         if code != 0:
             msg = ('{} `{}` returned {} '
@@ -467,14 +482,23 @@ class AmuletUtils(object):
             amulet.raise_status(amulet.FAIL, msg=msg)
         return str(output).split()
 
-    def get_unit_process_ids(self, unit_processes):
+    def get_unit_process_ids(self, unit_processes, expect_success=True):
         """Construct a dict containing unit sentries, process names, and
-        process IDs."""
+        process IDs.
+
+        :param unit_processes: A dictionary of Amulet sentry instance
+            to list of process names.
+        :param expect_success: if False expect the processes to not be
+            running, raise if they are.
+        :returns: Dictionary of Amulet sentry instance to dictionary
+            of process names to PIDs.
+        """
         pid_dict = {}
-        for sentry_unit, process_list in unit_processes.iteritems():
+        for sentry_unit, process_list in six.iteritems(unit_processes):
             pid_dict[sentry_unit] = {}
             for process in process_list:
-                pids = self.get_process_id_list(sentry_unit, process)
+                pids = self.get_process_id_list(
+                    sentry_unit, process, expect_success=expect_success)
                 pid_dict[sentry_unit].update({process: pids})
         return pid_dict
 
@@ -488,7 +512,7 @@ class AmuletUtils(object):
             return ('Unit count mismatch.  expected, actual: {}, '
                     '{} '.format(len(expected), len(actual)))
 
-        for (e_sentry, e_proc_names) in expected.iteritems():
+        for (e_sentry, e_proc_names) in six.iteritems(expected):
             e_sentry_name = e_sentry.info['unit_name']
             if e_sentry in actual.keys():
                 a_proc_names = actual[e_sentry]
@@ -507,11 +531,23 @@ class AmuletUtils(object):
                             '{}'.format(e_proc_name, a_proc_name))
 
                 a_pids_length = len(a_pids)
-                if e_pids_length != a_pids_length:
+                fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
-                    return ('PID count mismatch. {} ({}) expected, actual: '
                             '{}, {} ({})'.format(e_sentry_name, e_proc_name,
                                                  e_pids_length, a_pids_length,
                                                  a_pids))
+
+                # If expected is not bool, ensure PID quantities match
+                if not isinstance(e_pids_length, bool) and \
+                        a_pids_length != e_pids_length:
+                    return fail_msg
+                # If expected is bool True, ensure 1 or more PIDs exist
+                elif isinstance(e_pids_length, bool) and \
+                        e_pids_length is True and a_pids_length < 1:
+                    return fail_msg
+                # If expected is bool False, ensure 0 PIDs exist
+                elif isinstance(e_pids_length, bool) and \
+                        e_pids_length is False and a_pids_length != 0:
+                    return fail_msg
                 else:
                     self.log.debug('PID check OK: {} {} {}: '
                                    '{}'.format(e_sentry_name, e_proc_name,
@@ -531,3 +567,30 @@ class AmuletUtils(object):
             return 'Dicts within list are not identical'
 
         return None
+
+    def run_action(self, unit_sentry, action,
+                   _check_output=subprocess.check_output):
+        """Run the named action on a given unit sentry.
+
+        _check_output parameter is used for dependency injection.
+
+        @return action_id.
+        """
+        unit_id = unit_sentry.info["unit_name"]
+        command = ["juju", "action", "do", "--format=json", unit_id, action]
+        self.log.info("Running command: %s\n" % " ".join(command))
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        action_id = data[u'Action queued with id']
+        return action_id
+
+    def wait_on_action(self, action_id, _check_output=subprocess.check_output):
+        """Wait for a given action, returning if it completed or not.
+
+        _check_output parameter is used for dependency injection.
+        """
+        command = ["juju", "action", "fetch", "--format=json", "--wait=0",
+                   action_id]
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        return data.get(u"status") == "completed"

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+        base_charms = ['mysql', 'mongodb', 'nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
@@ -81,7 +81,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                       'ceph-osd', 'ceph-radosgw']
         # Most OpenStack subordinate charms do not expose an origin option
         # as that is controlled by the principle.
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services: