Update charm-helpers-hooks.yaml and sync ch

Using the new version of the sync tool which removes the charmhelpers
directory before syncing, run charm helpers sync to find any unexpected
missing dependencies.

Change-Id: I8e046f8816fd91920ba2c15f8c960b74442eb7bd

charm-helpers-hooks.yaml

@@ -13,6 +13,6 @@ include:
     - payload.execd
     - contrib.network.ip
     - contrib.network.ufw
-    - contrib.python.packages
+    - contrib.python
     - contrib.charmsupport
     - contrib.hardening|inc=*

charmhelpers/contrib/hahelpers/ceph.py

@@ -1,291 +0,0 @@
-#
-# Copyright 2012 Canonical Ltd.
-#
-# This file is sourced from lp:openstack-charm-helpers
-#
-# Authors:
-#  James Page <james.page@ubuntu.com>
-#  Adam Gandelman <adamg@ubuntu.com>
-#
-
-import commands
-import os
-import shutil
-import time
-
-from subprocess import (
-    check_call,
-    check_output,
-    CalledProcessError
-)
-
-from charmhelpers.core.hookenv import (
-    relation_get,
-    relation_ids,
-    related_units,
-    log,
-    INFO,
-    ERROR
-)
-
-from charmhelpers.core.host import (
-    apt_install,
-    mount,
-    mounts,
-    service_start,
-    service_stop,
-    umount,
-)
-
-KEYRING = '/etc/ceph/ceph.client.%s.keyring'
-KEYFILE = '/etc/ceph/ceph.client.%s.key'
-
-CEPH_CONF = """[global]
- auth supported = %(auth)s
- keyring = %(keyring)s
- mon host = %(mon_hosts)s
-"""
-
-
-def running(service):
-    # this local util can be dropped as soon the following branch lands
-    # in lp:charm-helpers
-    # https://code.launchpad.net/~gandelman-a/charm-helpers/service_running/
-    try:
-        output = check_output(['service', service, 'status'])
-    except CalledProcessError:
-        return False
-    else:
-        if ("start/running" in output or "is running" in output):
-            return True
-        else:
-            return False
-
-
-def install():
-    ceph_dir = "/etc/ceph"
-    if not os.path.isdir(ceph_dir):
-        os.mkdir(ceph_dir)
-    apt_install('ceph-common', fatal=True)
-
-
-def rbd_exists(service, pool, rbd_img):
-    (rc, out) = commands.getstatusoutput('rbd list --id %s --pool %s' %
-                                         (service, pool))
-    return rbd_img in out
-
-
-def create_rbd_image(service, pool, image, sizemb):
-    cmd = [
-        'rbd',
-        'create',
-        image,
-        '--size',
-        str(sizemb),
-        '--id',
-        service,
-        '--pool',
-        pool
-    ]
-    check_call(cmd)
-
-
-def pool_exists(service, name):
-    (rc, out) = commands.getstatusoutput("rados --id %s lspools" % service)
-    return name in out
-
-
-def create_pool(service, name):
-    cmd = [
-        'rados',
-        '--id',
-        service,
-        'mkpool',
-        name
-    ]
-    check_call(cmd)
-
-
-def keyfile_path(service):
-    return KEYFILE % service
-
-
-def keyring_path(service):
-    return KEYRING % service
-
-
-def create_keyring(service, key):
-    keyring = keyring_path(service)
-    if os.path.exists(keyring):
-        log('ceph: Keyring exists at %s.' % keyring, level=INFO)
-    cmd = [
-        'ceph-authtool',
-        keyring,
-        '--create-keyring',
-        '--name=client.%s' % service,
-        '--add-key=%s' % key
-    ]
-    check_call(cmd)
-    log('ceph: Created new ring at %s.' % keyring, level=INFO)
-
-
-def create_key_file(service, key):
-    # create a file containing the key
-    keyfile = keyfile_path(service)
-    if os.path.exists(keyfile):
-        log('ceph: Keyfile exists at %s.' % keyfile, level=INFO)
-    fd = open(keyfile, 'w')
-    fd.write(key)
-    fd.close()
-    log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
-
-
-def get_ceph_nodes():
-    hosts = []
-    for r_id in relation_ids('ceph'):
-        for unit in related_units(r_id):
-            hosts.append(relation_get('private-address', unit=unit, rid=r_id))
-    return hosts
-
-
-def configure(service, key, auth):
-    create_keyring(service, key)
-    create_key_file(service, key)
-    hosts = get_ceph_nodes()
-    mon_hosts = ",".join(map(str, hosts))
-    keyring = keyring_path(service)
-    with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
-        ceph_conf.write(CEPH_CONF % locals())
-    modprobe_kernel_module('rbd')
-
-
-def image_mapped(image_name):
-    (rc, out) = commands.getstatusoutput('rbd showmapped')
-    return image_name in out
-
-
-def map_block_storage(service, pool, image):
-    cmd = [
-        'rbd',
-        'map',
-        '%s/%s' % (pool, image),
-        '--user',
-        service,
-        '--secret',
-        keyfile_path(service),
-    ]
-    check_call(cmd)
-
-
-def filesystem_mounted(fs):
-    return fs in [f for m, f in mounts()]
-
-
-def make_filesystem(blk_device, fstype='ext4', timeout=10):
-    count = 0
-    e_noent = os.errno.ENOENT
-    while not os.path.exists(blk_device):
-        if count >= timeout:
-            log('ceph: gave up waiting on block device %s' % blk_device,
-                level=ERROR)
-            raise IOError(e_noent, os.strerror(e_noent), blk_device)
-        log('ceph: waiting for block device %s to appear' % blk_device,
-            level=INFO)
-        count += 1
-        time.sleep(1)
-    else:
-        log('ceph: Formatting block device %s as filesystem %s.' %
-            (blk_device, fstype), level=INFO)
-        check_call(['mkfs', '-t', fstype, blk_device])
-
-
-def place_data_on_ceph(service, blk_device, data_src_dst, fstype='ext4'):
-    # mount block device into /mnt
-    mount(blk_device, '/mnt')
-
-    # copy data to /mnt
-    try:
-        copy_files(data_src_dst, '/mnt')
-    except:
-        pass
-
-    # umount block device
-    umount('/mnt')
-
-    _dir = os.stat(data_src_dst)
-    uid = _dir.st_uid
-    gid = _dir.st_gid
-
-    # re-mount where the data should originally be
-    mount(blk_device, data_src_dst, persist=True)
-
-    # ensure original ownership of new mount.
-    cmd = ['chown', '-R', '%s:%s' % (uid, gid), data_src_dst]
-    check_call(cmd)
-
-
-# TODO: re-use
-def modprobe_kernel_module(module):
-    log('ceph: Loading kernel module', level=INFO)
-    cmd = ['modprobe', module]
-    check_call(cmd)
-    cmd = 'echo %s >> /etc/modules' % module
-    check_call(cmd, shell=True)
-
-
-def copy_files(src, dst, symlinks=False, ignore=None):
-    for item in os.listdir(src):
-        s = os.path.join(src, item)
-        d = os.path.join(dst, item)
-        if os.path.isdir(s):
-            shutil.copytree(s, d, symlinks, ignore)
-        else:
-            shutil.copy2(s, d)
-
-
-def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
-                        blk_device, fstype, system_services=[]):
-    """
-    To be called from the current cluster leader.
-    Ensures given pool and RBD image exists, is mapped to a block device,
-    and the device is formatted and mounted at the given mount_point.
-
-    If formatting a device for the first time, data existing at mount_point
-    will be migrated to the RBD device before being remounted.
-
-    All services listed in system_services will be stopped prior to data
-    migration and restarted when complete.
-    """
-    # Ensure pool, RBD image, RBD mappings are in place.
-    if not pool_exists(service, pool):
-        log('ceph: Creating new pool %s.' % pool, level=INFO)
-        create_pool(service, pool)
-
-    if not rbd_exists(service, pool, rbd_img):
-        log('ceph: Creating RBD image (%s).' % rbd_img, level=INFO)
-        create_rbd_image(service, pool, rbd_img, sizemb)
-
-    if not image_mapped(rbd_img):
-        log('ceph: Mapping RBD Image as a Block Device.', level=INFO)
-        map_block_storage(service, pool, rbd_img)
-
-    # make file system
-    # TODO: What happens if for whatever reason this is run again and
-    # the data is already in the rbd device and/or is mounted??
-    # When it is mounted already, it will fail to make the fs
-    # XXX: This is really sketchy!  Need to at least add an fstab entry
-    #      otherwise this hook will blow away existing data if its executed
-    #      after a reboot.
-    if not filesystem_mounted(mount_point):
-        make_filesystem(blk_device, fstype)
-
-        for svc in system_services:
-            if running(svc):
-                log('Stopping services %s prior to migrating data.' % svc,
-                    level=INFO)
-                service_stop(svc)
-
-        place_data_on_ceph(service, blk_device, mount_point, fstype)
-
-        for svc in system_services:
-            service_start(svc)

charmhelpers/contrib/openstack/amulet/utils.py

@@ -88,14 +88,14 @@ class OpenStackAmuletUtils(AmuletUtils):
         validation_function = self.validate_v2_endpoint_data
         xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens')
         if openstack_release and openstack_release >= xenial_queens:
-                validation_function = self.validate_v3_endpoint_data
+            validation_function = self.validate_v3_endpoint_data
-                expected = {
+            expected = {
-                    'id': expected['id'],
+                'id': expected['id'],
-                    'region': expected['region'],
+                'region': expected['region'],
-                    'region_id': 'RegionOne',
+                'region_id': 'RegionOne',
-                    'url': self.valid_url,
+                'url': self.valid_url,
-                    'interface': self.not_null,
+                'interface': self.not_null,
-                    'service_id': expected['service_id']}
+                'service_id': expected['service_id']}
         return validation_function(endpoints, admin_port, internal_port,
                                    public_port, expected)
 

charmhelpers/contrib/openstack/context.py

@@ -1427,11 +1427,11 @@ class ZeroMQContext(OSContextGenerator):
         ctxt = {}
         if is_relation_made('zeromq-configuration', 'host'):
             for rid in relation_ids('zeromq-configuration'):
-                    for unit in related_units(rid):
+                for unit in related_units(rid):
-                        ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
+                    ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
-                        ctxt['zmq_host'] = relation_get('host', unit, rid)
+                    ctxt['zmq_host'] = relation_get('host', unit, rid)
-                        ctxt['zmq_redis_address'] = relation_get(
+                    ctxt['zmq_redis_address'] = relation_get(
-                            'zmq_redis_address', unit, rid)
+                        'zmq_redis_address', unit, rid)
 
         return ctxt
 

charmhelpers/contrib/openstack/templating.py

@@ -183,7 +183,7 @@ class OSConfigRenderer(object):
         /tmp/templates/grizzly/api-paste.ini
         /tmp/templates/havana/api-paste.ini
 
-    Since it was registered with the grizzly release, it first seraches
+    Since it was registered with the grizzly release, it first searches
     the grizzly directory for nova.conf, then the templates dir.
 
     When writing api-paste.ini, it will find the template in the grizzly

charmhelpers/contrib/openstack/utils.py

@@ -83,7 +83,8 @@ from charmhelpers.fetch import (
     add_source as fetch_add_source,
     SourceConfigError,
     GPGKeyError,
-    get_upstream_version
+    get_upstream_version,
+    filter_missing_packages
 )
 
 from charmhelpers.fetch.snap import (
@@ -309,6 +310,15 @@ def error_out(msg):
     sys.exit(1)
 
 
+def get_installed_semantic_versioned_packages():
+    '''Get a list of installed packages which have OpenStack semantic versioning
+
+    :returns List of installed packages
+    :rtype: [pkg1, pkg2, ...]
+    '''
+    return filter_missing_packages(PACKAGE_CODENAMES.keys())
+
+
 def get_os_codename_install_source(src):
     '''Derive OpenStack release codename from a given installation source.'''
     ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
@@ -972,7 +982,9 @@ def _ows_check_charm_func(state, message, charm_func_with_configs):
     """
     if charm_func_with_configs:
         charm_state, charm_message = charm_func_with_configs()
-        if charm_state != 'active' and charm_state != 'unknown':
+        if (charm_state != 'active' and
+                charm_state != 'unknown' and
+                charm_state is not None):
             state = workload_state_compare(state, charm_state)
             if message:
                 charm_message = charm_message.replace("Incomplete relations: ",
@@ -1241,7 +1253,7 @@ def remote_restart(rel_name, remote_service=None):
 
 
 def check_actually_paused(services=None, ports=None):
-    """Check that services listed in the services object and and ports
+    """Check that services listed in the services object and ports
     are actually closed (not listened to), to verify that the unit is
     properly paused.
 

charmhelpers/contrib/python.py

@@ -0,0 +1,21 @@
+# Copyright 2014-2019 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+# deprecated aliases for backwards compatibility
+from charmhelpers.fetch.python import debug  # noqa
+from charmhelpers.fetch.python import packages  # noqa
+from charmhelpers.fetch.python import rpdb  # noqa
+from charmhelpers.fetch.python import version  # noqa

charmhelpers/contrib/storage/linux/ceph.py

@@ -856,12 +856,22 @@ def _keyring_path(service):
     return KEYRING.format(service)
 
 
-def create_keyring(service, key):
+def add_key(service, key):
-    """Create a new Ceph keyring containing key."""
+    """
+    Add a key to a keyring.
+
+    Creates the keyring if it doesn't already exist.
+
+    Logs and returns if the key is already in the keyring.
+    """
     keyring = _keyring_path(service)
     if os.path.exists(keyring):
-        log('Ceph keyring exists at %s.' % keyring, level=WARNING)
+        with open(keyring, 'r') as ring:
-        return
+            if key in ring.read():
+                log('Ceph keyring exists at %s and has not changed.' % keyring,
+                    level=DEBUG)
+                return
+            log('Updating existing keyring %s.' % keyring, level=DEBUG)
 
     cmd = ['ceph-authtool', keyring, '--create-keyring',
            '--name=client.{}'.format(service), '--add-key={}'.format(key)]
@@ -869,6 +879,11 @@ def create_keyring(service, key):
     log('Created new ceph keyring at %s.' % keyring, level=DEBUG)
 
 
+def create_keyring(service, key):
+    """Deprecated. Please use the more accurately named 'add_key'"""
+    return add_key(service, key)
+
+
 def delete_keyring(service):
     """Delete an existing Ceph keyring."""
     keyring = _keyring_path(service)
@@ -905,7 +920,7 @@ def get_ceph_nodes(relation='ceph'):
 
 def configure(service, key, auth, use_syslog):
     """Perform basic configuration of Ceph."""
-    create_keyring(service, key)
+    add_key(service, key)
     create_key_file(service, key)
     hosts = get_ceph_nodes()
     with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
@@ -1068,7 +1083,7 @@ def ensure_ceph_keyring(service, user=None, group=None,
     if not key:
         return False
 
-    create_keyring(service=service, key=key)
+    add_key(service=service, key=key)
     keyring = _keyring_path(service)
     if user and group:
         check_call(['chown', '%s.%s' % (user, group), keyring])

charmhelpers/core/host.py

@@ -46,6 +46,7 @@ if __platform__ == "ubuntu":
         lsb_release,
         cmp_pkgrevno,
         CompareHostReleases,
+        get_distrib_codename,
     )  # flake8: noqa -- ignore F401 for this import
 elif __platform__ == "centos":
     from charmhelpers.core.host_factory.centos import (  # NOQA:F401

charmhelpers/core/host_factory/ubuntu.py

@@ -72,6 +72,14 @@ def lsb_release():
     return d
 
 
+def get_distrib_codename():
+    """Return the codename of the distribution
+    :returns: The codename
+    :rtype: str
+    """
+    return lsb_release()['DISTRIB_CODENAME'].lower()
+
+
 def cmp_pkgrevno(package, revno, pkgcache=None):
     """Compare supplied revno with the revno of the installed package.
 

charmhelpers/fetch/python/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-2015 Canonical Limited.
+# Copyright 2014-2019 Canonical Limited.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

charmhelpers/fetch/python/debug.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import atexit
+import sys
+
+from charmhelpers.fetch.python.rpdb import Rpdb
+from charmhelpers.core.hookenv import (
+    open_port,
+    close_port,
+    ERROR,
+    log
+)
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+
+DEFAULT_ADDR = "0.0.0.0"
+DEFAULT_PORT = 4444
+
+
+def _error(message):
+    log(message, level=ERROR)
+
+
+def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT):
+    """
+    Set a trace point using the remote debugger
+    """
+    atexit.register(close_port, port)
+    try:
+        log("Starting a remote python debugger session on %s:%s" % (addr,
+                                                                    port))
+        open_port(port)
+        debugger = Rpdb(addr=addr, port=port)
+        debugger.set_trace(sys._getframe().f_back)
+    except Exception:
+        _error("Cannot start a remote debug session on %s:%s" % (addr,
+                                                                 port))

charmhelpers/fetch/python/rpdb.py

@@ -0,0 +1,56 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Remote Python Debugger (pdb wrapper)."""
+
+import pdb
+import socket
+import sys
+
+__author__ = "Bertrand Janin <b@janin.com>"
+__version__ = "0.1.3"
+
+
+class Rpdb(pdb.Pdb):
+
+    def __init__(self, addr="127.0.0.1", port=4444):
+        """Initialize the socket and initialize pdb."""
+
+        # Backup stdin and stdout before replacing them by the socket handle
+        self.old_stdout = sys.stdout
+        self.old_stdin = sys.stdin
+
+        # Open a 'reusable' socket to let the webapp reload on the same port
+        self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+        self.skt.bind((addr, port))
+        self.skt.listen(1)
+        (clientsocket, address) = self.skt.accept()
+        handle = clientsocket.makefile('rw')
+        pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle)
+        sys.stdout = sys.stdin = handle
+
+    def shutdown(self):
+        """Revert stdin and stdout, close the socket."""
+        sys.stdout = self.old_stdout
+        sys.stdin = self.old_stdin
+        self.skt.close()
+        self.set_continue()
+
+    def do_continue(self, arg):
+        """Stop all operation on ``continue``."""
+        self.shutdown()
+        return 1
+
+    do_EOF = do_quit = do_exit = do_c = do_cont = do_continue

charmhelpers/fetch/python/version.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+
+
+def current_version():
+    """Current system python version"""
+    return sys.version_info
+
+
+def current_version_string():
+    """Current system python version as string major.minor.micro"""
+    return "{0}.{1}.{2}".format(sys.version_info.major,
+                                sys.version_info.minor,
+                                sys.version_info.micro)