Sync charm-helpers

hooks/charmhelpers/contrib/network/ip.py

@@ -17,13 +17,16 @@
 import glob
 import re
 import subprocess
+import six
+import socket
 
 from functools import partial
 
 from charmhelpers.core.hookenv import unit_get
 from charmhelpers.fetch import apt_install
 from charmhelpers.core.hookenv import (
-    log
+    log,
+    WARNING,
 )
 
 try:
@@ -365,3 +368,83 @@ def is_bridge_member(nic):
             return True
 
     return False
+
+
+def is_ip(address):
+    """
+    Returns True if address is a valid IP address.
+    """
+    try:
+        # Test to see if already an IPv4 address
+        socket.inet_aton(address)
+        return True
+    except socket.error:
+        return False
+
+
+def ns_query(address):
+    try:
+        import dns.resolver
+    except ImportError:
+        apt_install('python-dnspython')
+        import dns.resolver
+
+    if isinstance(address, dns.name.Name):
+        rtype = 'PTR'
+    elif isinstance(address, six.string_types):
+        rtype = 'A'
+    else:
+        return None
+
+    answers = dns.resolver.query(address, rtype)
+    if answers:
+        return str(answers[0])
+    return None
+
+
+def get_host_ip(hostname, fallback=None):
+    """
+    Resolves the IP for a given hostname, or returns
+    the input if it is already an IP.
+    """
+    if is_ip(hostname):
+        return hostname
+
+    ip_addr = ns_query(hostname)
+    if not ip_addr:
+        try:
+            ip_addr = socket.gethostbyname(hostname)
+        except:
+            log("Failed to resolve hostname '%s'" % (hostname),
+                level=WARNING)
+            return fallback
+    return ip_addr
+
+
+def get_hostname(address, fqdn=True):
+    """
+    Resolves hostname for given IP, or returns the input
+    if it is already a hostname.
+    """
+    if is_ip(address):
+        try:
+            import dns.reversename
+        except ImportError:
+            apt_install("python-dnspython")
+            import dns.reversename
+
+        rev = dns.reversename.from_address(address)
+        result = ns_query(rev)
+        if not result:
+            return None
+    else:
+        result = address
+
+    if fqdn:
+        # strip trailing .
+        if result.endswith('.'):
+            return result[:-1]
+        else:
+            return result
+    else:
+        return result.split('.')[0]

hooks/charmhelpers/contrib/openstack/context.py

@@ -21,6 +21,7 @@ from base64 import b64decode
 from subprocess import check_call
 
 import six
+import yaml
 
 from charmhelpers.fetch import (
     apt_install,
@@ -104,9 +105,41 @@ def context_complete(ctxt):
 def config_flags_parser(config_flags):
     """Parses config flags string into dict.
 
+    This parsing method supports a few different formats for the config
+    flag values to be parsed:
+
+      1. A string in the simple format of key=value pairs, with the possibility
+         of specifying multiple key value pairs within the same string. For
+         example, a string in the format of 'key1=value1, key2=value2' will
+         return a dict of:
+         {'key1': 'value1',
+          'key2': 'value2'}.
+
+      2. A string in the above format, but supporting a comma-delimited list
+         of values for the same key. For example, a string in the format of
+         'key1=value1, key2=value3,value4,value5' will return a dict of:
+         {'key1', 'value1',
+          'key2', 'value2,value3,value4'}
+
+      3. A string containing a colon character (:) prior to an equal
+         character (=) will be treated as yaml and parsed as such. This can be
+         used to specify more complex key value pairs. For example,
+         a string in the format of 'key1: subkey1=value1, subkey2=value2' will
+         return a dict of:
+         {'key1', 'subkey1=value1, subkey2=value2'}
+
     The provided config_flags string may be a list of comma-separated values
     which themselves may be comma-separated list of values.
     """
+    # If we find a colon before an equals sign then treat it as yaml.
+    # Note: limit it to finding the colon first since this indicates assignment
+    # for inline yaml.
+    colon = config_flags.find(':')
+    equals = config_flags.find('=')
+    if colon > 0:
+        if colon < equals or equals < 0:
+            return yaml.safe_load(config_flags)
+
     if config_flags.find('==') >= 0:
         log("config_flags is not in expected format (key=value)", level=ERROR)
         raise OSContextError
@@ -191,7 +224,7 @@ class SharedDBContext(OSContextGenerator):
                                         unit=local_unit())
             if set_hostname != access_hostname:
                 relation_set(relation_settings={hostname_key: access_hostname})
-                return ctxt  # Defer any further hook execution for now....
+                return None  # Defer any further hook execution for now....
 
         password_setting = 'password'
         if self.relation_prefix:

hooks/charmhelpers/contrib/openstack/utils.py

@@ -20,15 +20,18 @@
 from collections import OrderedDict
 from functools import wraps
 
+import errno
 import subprocess
 import json
 import os
-import socket
 import sys
+import time
 
 import six
 import yaml
 
+from charmhelpers.contrib.network import ip
+
 from charmhelpers.core.hookenv import (
     config,
     log as juju_log,
@@ -421,77 +424,10 @@ def clean_storage(block_device):
     else:
         zap_disk(block_device)
 
-
+is_ip = ip.is_ip
-def is_ip(address):
+ns_query = ip.ns_query
-    """
+get_host_ip = ip.get_host_ip
-    Returns True if address is a valid IP address.
+get_hostname = ip.get_hostname
-    """
-    try:
-        # Test to see if already an IPv4 address
-        socket.inet_aton(address)
-        return True
-    except socket.error:
-        return False
-
-
-def ns_query(address):
-    try:
-        import dns.resolver
-    except ImportError:
-        apt_install('python-dnspython')
-        import dns.resolver
-
-    if isinstance(address, dns.name.Name):
-        rtype = 'PTR'
-    elif isinstance(address, six.string_types):
-        rtype = 'A'
-    else:
-        return None
-
-    answers = dns.resolver.query(address, rtype)
-    if answers:
-        return str(answers[0])
-    return None
-
-
-def get_host_ip(hostname):
-    """
-    Resolves the IP for a given hostname, or returns
-    the input if it is already an IP.
-    """
-    if is_ip(hostname):
-        return hostname
-
-    return ns_query(hostname)
-
-
-def get_hostname(address, fqdn=True):
-    """
-    Resolves hostname for given IP, or returns the input
-    if it is already a hostname.
-    """
-    if is_ip(address):
-        try:
-            import dns.reversename
-        except ImportError:
-            apt_install('python-dnspython')
-            import dns.reversename
-
-        rev = dns.reversename.from_address(address)
-        result = ns_query(rev)
-        if not result:
-            return None
-    else:
-        result = address
-
-    if fqdn:
-        # strip trailing .
-        if result.endswith('.'):
-            return result[:-1]
-        else:
-            return result
-    else:
-        return result.split('.')[0]
 
 
 def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):
@@ -536,89 +472,115 @@ def os_requires_version(ostack_release, pkg):
 
 def git_install_requested():
     """Returns true if openstack-origin-git is specified."""
-    return config('openstack-origin-git') != "None"
+    return config('openstack-origin-git') != None
 
 
 requirements_dir = None
 
 
-def git_clone_and_install(file_name, core_project):
+def git_clone_and_install(projects, core_project,
-    """Clone/install all OpenStack repos specified in yaml config file."""
+                          parent_dir='/mnt/openstack-git'):
+    """Clone/install all OpenStack repos specified in projects dictionary."""
     global requirements_dir
+    update_reqs = True
 
-    if file_name == "None":
+    if not projects:
         return
 
-    yaml_file = os.path.join(charm_dir(), file_name)
-
     # clone/install the requirements project first
-    installed = _git_clone_and_install_subset(yaml_file,
+    installed = _git_clone_and_install_subset(projects, parent_dir,
                                               whitelist=['requirements'])
     if 'requirements' not in installed:
-        error_out('requirements git repository must be specified')
+        update_reqs = False
 
     # clone/install all other projects except requirements and the core project
     blacklist = ['requirements', core_project]
-    _git_clone_and_install_subset(yaml_file, blacklist=blacklist,
+    _git_clone_and_install_subset(projects, parent_dir, blacklist=blacklist,
-                                  update_requirements=True)
+                                  update_requirements=update_reqs)
 
     # clone/install the core project
     whitelist = [core_project]
-    installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
+    installed = _git_clone_and_install_subset(projects, parent_dir,
-                                              update_requirements=True)
+                                              whitelist=whitelist,
+                                              update_requirements=update_reqs)
     if core_project not in installed:
         error_out('{} git repository must be specified'.format(core_project))
 
 
-def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
+def _git_clone_and_install_subset(projects, parent_dir, whitelist=[],
-                                  update_requirements=False):
+                                  blacklist=[], update_requirements=False):
-    """Clone/install subset of OpenStack repos specified in yaml config file."""
+    """Clone/install subset of OpenStack repos specified in projects dict."""
     global requirements_dir
     installed = []
 
-    with open(yaml_file, 'r') as fd:
+    for proj, val in projects.items():
-        projects = yaml.load(fd)
+        # The project subset is chosen based on the following 3 rules:
-        for proj, val in projects.items():
+        # 1) If project is in blacklist, we don't clone/install it, period.
-            # The project subset is chosen based on the following 3 rules:
+        # 2) If whitelist is empty, we clone/install everything else.
-            # 1) If project is in blacklist, we don't clone/install it, period.
+        # 3) If whitelist is not empty, we clone/install everything in the
-            # 2) If whitelist is empty, we clone/install everything else.
+        #    whitelist.
-            # 3) If whitelist is not empty, we clone/install everything in the
+        if proj in blacklist:
-            #    whitelist.
+            continue
-            if proj in blacklist:
+        if whitelist and proj not in whitelist:
-                continue
+            continue
-            if whitelist and proj not in whitelist:
+        repo = val['repository']
-                continue
+        branch = val['branch']
-            repo = val['repository']
+        repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
-            branch = val['branch']
+                                                 update_requirements)
-            repo_dir = _git_clone_and_install_single(repo, branch,
+        if proj == 'requirements':
-                                                     update_requirements)
+            requirements_dir = repo_dir
-            if proj == 'requirements':
+        installed.append(proj)
-                requirements_dir = repo_dir
-            installed.append(proj)
     return installed
 
 
-def _git_clone_and_install_single(repo, branch, update_requirements=False):
+def _git_clone_and_install_single(repo, branch, parent_dir,
+                                  update_requirements=False):
     """Clone and install a single git repository."""
-    dest_parent_dir = "/mnt/openstack-git/"
+    dest_dir = os.path.join(parent_dir, os.path.basename(repo))
-    dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
+    lock_dir = os.path.join(parent_dir, os.path.basename(repo) + '.lock')
 
-    if not os.path.exists(dest_parent_dir):
+    # Note(coreycb): The parent directory for storing git repositories can be
-        juju_log('Host dir not mounted at {}. '
+    # shared by multiple charms via bind mount, etc, so we use exception
-                 'Creating directory there instead.'.format(dest_parent_dir))
+    # handling to ensure the test for existence and mkdir are atomic.
-        os.mkdir(dest_parent_dir)
+    try:
-
+        os.mkdir(parent_dir)
-    if not os.path.exists(dest_dir):
+    except OSError as e:
-        juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
+        if e.errno == errno.EEXIST:
-        repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
+            juju_log('Directory already exists at {}. '
+                     'No need to create directory.'.format(parent_dir))
+            pass
     else:
-        repo_dir = dest_dir
+        juju_log('Host directory not mounted at {}. '
+                 'Directory created.'.format(parent_dir))
 
-    if update_requirements:
+    # Note(coreycb): Similar to above, the cloned git repositories can be shared
-        if not requirements_dir:
+    # by multiple charms via bind mount, etc, so we use exception handling and
-            error_out('requirements repo must be cloned before '
+    # special lock directories to ensure that a repository clone is only
-                      'updating from global requirements.')
+    # attempted once.
-        _git_update_requirements(repo_dir, requirements_dir)
+    try:
+        os.mkdir(lock_dir)
+    except OSError as e:                                                                              
+        if e.errno == errno.EEXIST:
+            juju_log('Lock directory exists at {}. Skip git clone and wait '
+                     'for lock removal before installing.'.format(lock_dir))
+            while os.path.exists(lock_dir):
+                juju_log('Waiting for git clone to complete before installing.')
+                time.sleep(1)
+            pass
+    else:
+        if not os.path.exists(dest_dir):
+            juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
+            repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
+        else:
+            repo_dir = dest_dir
+
+        if update_requirements:
+            if not requirements_dir:
+                error_out('requirements repo must be cloned before '
+                          'updating from global requirements.')
+            _git_update_requirements(repo_dir, requirements_dir)
+
+        os.rmdir(lock_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     pip_install(repo_dir)

hooks/charmhelpers/core/services/helpers.py

@@ -45,12 +45,14 @@ class RelationContext(dict):
     """
     name = None
     interface = None
-    required_keys = []
 
     def __init__(self, name=None, additional_required_keys=None):
+        if not hasattr(self, 'required_keys'):
+            self.required_keys = []
+
         if name is not None:
             self.name = name
-        if additional_required_keys is not None:
+        if additional_required_keys:
             self.required_keys.extend(additional_required_keys)
         self.get_data()
 
@@ -134,7 +136,10 @@ class MysqlRelation(RelationContext):
     """
     name = 'db'
     interface = 'mysql'
-    required_keys = ['host', 'user', 'password', 'database']
+
+    def __init__(self, *args, **kwargs):
+        self.required_keys = ['host', 'user', 'password', 'database']
+        super(HttpRelation).__init__(self, *args, **kwargs)
 
 
 class HttpRelation(RelationContext):
@@ -146,7 +151,10 @@ class HttpRelation(RelationContext):
     """
     name = 'website'
     interface = 'http'
-    required_keys = ['host', 'port']
+
+    def __init__(self, *args, **kwargs):
+        self.required_keys = ['host', 'port']
+        super(HttpRelation).__init__(self, *args, **kwargs)
 
     def provide_data(self):
         return {