Update charm-helpers-hooks.yaml and sync ch

Using the new version of the sync tool which removes the charmhelpers
directory before syncing, run charm helpers sync to find any unexpected
missing dependencies.

Change-Id: I0be40a133391da18cbd2b70627228910d172d28d

charm-helpers-hooks.yaml

@@ -12,6 +12,6 @@ include:
     - payload.execd
     - payload.archive
     - contrib.network.ip
-    - contrib.python.packages
+    - contrib.python
     - contrib.charmsupport
     - contrib.hardening|inc=*

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -88,14 +88,14 @@ class OpenStackAmuletUtils(AmuletUtils):
         validation_function = self.validate_v2_endpoint_data
         xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens')
         if openstack_release and openstack_release >= xenial_queens:
-                validation_function = self.validate_v3_endpoint_data
-                expected = {
-                    'id': expected['id'],
-                    'region': expected['region'],
-                    'region_id': 'RegionOne',
-                    'url': self.valid_url,
-                    'interface': self.not_null,
-                    'service_id': expected['service_id']}
+            validation_function = self.validate_v3_endpoint_data
+            expected = {
+                'id': expected['id'],
+                'region': expected['region'],
+                'region_id': 'RegionOne',
+                'url': self.valid_url,
+                'interface': self.not_null,
+                'service_id': expected['service_id']}
         return validation_function(endpoints, admin_port, internal_port,
                                    public_port, expected)
 

hooks/charmhelpers/contrib/openstack/context.py

@@ -1427,11 +1427,11 @@ class ZeroMQContext(OSContextGenerator):
         ctxt = {}
         if is_relation_made('zeromq-configuration', 'host'):
             for rid in relation_ids('zeromq-configuration'):
-                    for unit in related_units(rid):
-                        ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
-                        ctxt['zmq_host'] = relation_get('host', unit, rid)
-                        ctxt['zmq_redis_address'] = relation_get(
-                            'zmq_redis_address', unit, rid)
+                for unit in related_units(rid):
+                    ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
+                    ctxt['zmq_host'] = relation_get('host', unit, rid)
+                    ctxt['zmq_redis_address'] = relation_get(
+                        'zmq_redis_address', unit, rid)
 
         return ctxt
 

hooks/charmhelpers/contrib/openstack/templating.py

@@ -183,7 +183,7 @@ class OSConfigRenderer(object):
         /tmp/templates/grizzly/api-paste.ini
         /tmp/templates/havana/api-paste.ini
 
-    Since it was registered with the grizzly release, it first seraches
+    Since it was registered with the grizzly release, it first searches
     the grizzly directory for nova.conf, then the templates dir.
 
     When writing api-paste.ini, it will find the template in the grizzly

hooks/charmhelpers/contrib/openstack/utils.py

@@ -83,7 +83,8 @@ from charmhelpers.fetch import (
     add_source as fetch_add_source,
     SourceConfigError,
     GPGKeyError,
-    get_upstream_version
+    get_upstream_version,
+    filter_missing_packages
 )
 
 from charmhelpers.fetch.snap import (
@@ -309,6 +310,15 @@ def error_out(msg):
     sys.exit(1)
 
 
+def get_installed_semantic_versioned_packages():
+    '''Get a list of installed packages which have OpenStack semantic versioning
+
+    :returns List of installed packages
+    :rtype: [pkg1, pkg2, ...]
+    '''
+    return filter_missing_packages(PACKAGE_CODENAMES.keys())
+
+
 def get_os_codename_install_source(src):
     '''Derive OpenStack release codename from a given installation source.'''
     ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
@@ -972,7 +982,9 @@ def _ows_check_charm_func(state, message, charm_func_with_configs):
     """
     if charm_func_with_configs:
         charm_state, charm_message = charm_func_with_configs()
-        if charm_state != 'active' and charm_state != 'unknown':
+        if (charm_state != 'active' and
+                charm_state != 'unknown' and
+                charm_state is not None):
             state = workload_state_compare(state, charm_state)
             if message:
                 charm_message = charm_message.replace("Incomplete relations: ",
@@ -1241,7 +1253,7 @@ def remote_restart(rel_name, remote_service=None):
 
 
 def check_actually_paused(services=None, ports=None):
-    """Check that services listed in the services object and and ports
+    """Check that services listed in the services object and ports
     are actually closed (not listened to), to verify that the unit is
     properly paused.
 

hooks/charmhelpers/contrib/python.py

@@ -0,0 +1,21 @@
+# Copyright 2014-2019 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+# deprecated aliases for backwards compatibility
+from charmhelpers.fetch.python import debug  # noqa
+from charmhelpers.fetch.python import packages  # noqa
+from charmhelpers.fetch.python import rpdb  # noqa
+from charmhelpers.fetch.python import version  # noqa

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -856,12 +856,22 @@ def _keyring_path(service):
     return KEYRING.format(service)
 
 
-def create_keyring(service, key):
-    """Create a new Ceph keyring containing key."""
+def add_key(service, key):
+    """
+    Add a key to a keyring.
+
+    Creates the keyring if it doesn't already exist.
+
+    Logs and returns if the key is already in the keyring.
+    """
     keyring = _keyring_path(service)
     if os.path.exists(keyring):
-        log('Ceph keyring exists at %s.' % keyring, level=WARNING)
-        return
+        with open(keyring, 'r') as ring:
+            if key in ring.read():
+                log('Ceph keyring exists at %s and has not changed.' % keyring,
+                    level=DEBUG)
+                return
+            log('Updating existing keyring %s.' % keyring, level=DEBUG)
 
     cmd = ['ceph-authtool', keyring, '--create-keyring',
            '--name=client.{}'.format(service), '--add-key={}'.format(key)]
@@ -869,6 +879,11 @@ def create_keyring(service, key):
     log('Created new ceph keyring at %s.' % keyring, level=DEBUG)
 
 
+def create_keyring(service, key):
+    """Deprecated. Please use the more accurately named 'add_key'"""
+    return add_key(service, key)
+
+
 def delete_keyring(service):
     """Delete an existing Ceph keyring."""
     keyring = _keyring_path(service)
@@ -905,7 +920,7 @@ def get_ceph_nodes(relation='ceph'):
 
 def configure(service, key, auth, use_syslog):
     """Perform basic configuration of Ceph."""
-    create_keyring(service, key)
+    add_key(service, key)
     create_key_file(service, key)
     hosts = get_ceph_nodes()
     with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
@@ -1068,7 +1083,7 @@ def ensure_ceph_keyring(service, user=None, group=None,
     if not key:
         return False
 
-    create_keyring(service=service, key=key)
+    add_key(service=service, key=key)
     keyring = _keyring_path(service)
     if user and group:
         check_call(['chown', '%s.%s' % (user, group), keyring])

hooks/charmhelpers/core/host.py

@@ -46,6 +46,7 @@ if __platform__ == "ubuntu":
         lsb_release,
         cmp_pkgrevno,
         CompareHostReleases,
+        get_distrib_codename,
     )  # flake8: noqa -- ignore F401 for this import
 elif __platform__ == "centos":
     from charmhelpers.core.host_factory.centos import (  # NOQA:F401

hooks/charmhelpers/core/host_factory/ubuntu.py

@@ -72,6 +72,14 @@ def lsb_release():
     return d
 
 
+def get_distrib_codename():
+    """Return the codename of the distribution
+    :returns: The codename
+    :rtype: str
+    """
+    return lsb_release()['DISTRIB_CODENAME'].lower()
+
+
 def cmp_pkgrevno(package, revno, pkgcache=None):
     """Compare supplied revno with the revno of the installed package.
 

hooks/charmhelpers/fetch/python/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-2015 Canonical Limited.
+# Copyright 2014-2019 Canonical Limited.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

hooks/charmhelpers/fetch/python/debug.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import atexit
+import sys
+
+from charmhelpers.fetch.python.rpdb import Rpdb
+from charmhelpers.core.hookenv import (
+    open_port,
+    close_port,
+    ERROR,
+    log
+)
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+
+DEFAULT_ADDR = "0.0.0.0"
+DEFAULT_PORT = 4444
+
+
+def _error(message):
+    log(message, level=ERROR)
+
+
+def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT):
+    """
+    Set a trace point using the remote debugger
+    """
+    atexit.register(close_port, port)
+    try:
+        log("Starting a remote python debugger session on %s:%s" % (addr,
+                                                                    port))
+        open_port(port)
+        debugger = Rpdb(addr=addr, port=port)
+        debugger.set_trace(sys._getframe().f_back)
+    except Exception:
+        _error("Cannot start a remote debug session on %s:%s" % (addr,
+                                                                 port))

hooks/charmhelpers/fetch/python/rpdb.py

@@ -0,0 +1,56 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Remote Python Debugger (pdb wrapper)."""
+
+import pdb
+import socket
+import sys
+
+__author__ = "Bertrand Janin <b@janin.com>"
+__version__ = "0.1.3"
+
+
+class Rpdb(pdb.Pdb):
+
+    def __init__(self, addr="127.0.0.1", port=4444):
+        """Initialize the socket and initialize pdb."""
+
+        # Backup stdin and stdout before replacing them by the socket handle
+        self.old_stdout = sys.stdout
+        self.old_stdin = sys.stdin
+
+        # Open a 'reusable' socket to let the webapp reload on the same port
+        self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+        self.skt.bind((addr, port))
+        self.skt.listen(1)
+        (clientsocket, address) = self.skt.accept()
+        handle = clientsocket.makefile('rw')
+        pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle)
+        sys.stdout = sys.stdin = handle
+
+    def shutdown(self):
+        """Revert stdin and stdout, close the socket."""
+        sys.stdout = self.old_stdout
+        sys.stdin = self.old_stdin
+        self.skt.close()
+        self.set_continue()
+
+    def do_continue(self, arg):
+        """Stop all operation on ``continue``."""
+        self.shutdown()
+        return 1
+
+    do_EOF = do_quit = do_exit = do_c = do_cont = do_continue

hooks/charmhelpers/fetch/python/version.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+
+
+def current_version():
+    """Current system python version"""
+    return sys.version_info
+
+
+def current_version_string():
+    """Current system python version as string major.minor.micro"""
+    return "{0}.{1}.{2}".format(sys.version_info.major,
+                                sys.version_info.minor,
+                                sys.version_info.micro)