merge from trunk

Makefile

@@ -12,6 +12,6 @@ test:
 sync:
 	@charm-helper-sync -c charm-helpers.yaml
 
-publish:
+publish: lint test
 	bzr push lp:charms/keystone
 	bzr push lp:charms/trusty/keystone

hooks/charmhelpers/contrib/openstack/context.py

@@ -570,7 +570,7 @@ class NeutronContext(OSContextGenerator):
 
         if self.plugin == 'ovs':
             ctxt.update(self.ovs_ctxt())
-        elif self.plugin == 'nvp':
+        elif self.plugin in ['nvp', 'nsx']:
             ctxt.update(self.nvp_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')

hooks/charmhelpers/contrib/openstack/neutron.py

@@ -114,14 +114,30 @@ def neutron_plugins():
             'server_packages': ['neutron-server',
                                 'neutron-plugin-nicira'],
             'server_services': ['neutron-server']
+        },
+        'nsx': {
+            'config': '/etc/neutron/plugins/vmware/nsx.ini',
+            'driver': 'vmware',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [],
+            'server_packages': ['neutron-server',
+                                'neutron-plugin-vmware'],
+            'server_services': ['neutron-server']
         }
     }
-    # NOTE: patch in ml2 plugin for icehouse onwards
     if release >= 'icehouse':
+        # NOTE: patch in ml2 plugin for icehouse onwards
         plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
         plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
         plugins['ovs']['server_packages'] = ['neutron-server',
                                              'neutron-plugin-ml2']
+        # NOTE: patch in vmware renames nvp->nsx for icehouse onwards
+        plugins['nvp'] = plugins['nsx']
     return plugins
 
 

hooks/charmhelpers/contrib/openstack/utils.py

@@ -131,6 +131,11 @@ def get_os_version_codename(codename):
 def get_os_codename_package(package, fatal=True):
     '''Derive OpenStack release codename from an installed package.'''
     apt.init()
+
+    # Tell apt to build an in-memory cache to prevent race conditions (if
+    # another process is already building the cache).
+    apt.config.set("Dir::Cache::pkgcache", "")
+
     cache = apt.Cache()
 
     try:
@@ -183,7 +188,7 @@ def get_os_version_package(pkg, fatal=True):
         if cname == codename:
             return version
     #e = "Could not determine OpenStack version for package: %s" % pkg
-    #error_out(e)
+    # error_out(e)
 
 
 os_rel = None
@@ -401,6 +406,8 @@ def ns_query(address):
         rtype = 'PTR'
     elif isinstance(address, basestring):
         rtype = 'A'
+    else:
+        return None
 
     answers = dns.resolver.query(address, rtype)
     if answers:

hooks/charmhelpers/contrib/storage/linux/lvm.py

@@ -62,7 +62,7 @@ def list_lvm_volume_group(block_device):
     pvd = check_output(['pvdisplay', block_device]).splitlines()
     for l in pvd:
         if l.strip().startswith('VG Name'):
-            vg = ' '.join(l.split()).split(' ').pop()
+            vg = ' '.join(l.strip().split()[2:])
     return vg
 
 

hooks/charmhelpers/contrib/storage/linux/utils.py

@@ -1,8 +1,11 @@
-from os import stat
+import os
+import re
 from stat import S_ISBLK
 
 from subprocess import (
-    check_call
+    check_call,
+    check_output,
+    call
 )
 
 
@@ -12,7 +15,9 @@ def is_block_device(path):
 
     :returns: boolean: True if path is a block device, False if not.
     '''
-    return S_ISBLK(stat(path).st_mode)
+    if not os.path.exists(path):
+        return False
+    return S_ISBLK(os.stat(path).st_mode)
 
 
 def zap_disk(block_device):
@@ -22,5 +27,23 @@ def zap_disk(block_device):
 
     :param block_device: str: Full path of block device to clean.
     '''
-    check_call(['sgdisk', '--zap-all', '--clear',
+    # sometimes sgdisk exits non-zero; this is OK, dd will clean up
-                '--mbrtogpt', block_device])
+    call(['sgdisk', '--zap-all', '--mbrtogpt',
+          '--clear', block_device])
+    dev_end = check_output(['blockdev', '--getsz', block_device])
+    gpt_end = int(dev_end.split()[0]) - 100
+    check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
+                'bs=1M', 'count=1'])
+    check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
+                'bs=512', 'count=100', 'seek=%s' % (gpt_end)])
+
+def is_device_mounted(device):
+    '''Given a device path, return True if that device is mounted, and False
+    if it isn't.
+
+    :param device: str: Full path of the device to check.
+    :returns: boolean: True if the path represents a mounted device, False if
+        it doesn't.
+    '''
+    out = check_output(['mount'])
+    return bool(re.search(device + r"[0-9]+\b", out))

hooks/charmhelpers/core/hookenv.py

@@ -155,6 +155,100 @@ def hook_name():
     return os.path.basename(sys.argv[0])
 
 
+class Config(dict):
+    """A Juju charm config dictionary that can write itself to
+    disk (as json) and track which values have changed since
+    the previous hook invocation.
+
+    Do not instantiate this object directly - instead call
+    ``hookenv.config()``
+
+    Example usage::
+
+        >>> # inside a hook
+        >>> from charmhelpers.core import hookenv
+        >>> config = hookenv.config()
+        >>> config['foo']
+        'bar'
+        >>> config['mykey'] = 'myval'
+        >>> config.save()
+
+
+        >>> # user runs `juju set mycharm foo=baz`
+        >>> # now we're inside subsequent config-changed hook
+        >>> config = hookenv.config()
+        >>> config['foo']
+        'baz'
+        >>> # test to see if this val has changed since last hook
+        >>> config.changed('foo')
+        True
+        >>> # what was the previous value?
+        >>> config.previous('foo')
+        'bar'
+        >>> # keys/values that we add are preserved across hooks
+        >>> config['mykey']
+        'myval'
+        >>> # don't forget to save at the end of hook!
+        >>> config.save()
+
+    """
+    CONFIG_FILE_NAME = '.juju-persistent-config'
+
+    def __init__(self, *args, **kw):
+        super(Config, self).__init__(*args, **kw)
+        self._prev_dict = None
+        self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
+        if os.path.exists(self.path):
+            self.load_previous()
+
+    def load_previous(self, path=None):
+        """Load previous copy of config from disk so that current values
+        can be compared to previous values.
+
+        :param path:
+
+            File path from which to load the previous config. If `None`,
+            config is loaded from the default location. If `path` is
+            specified, subsequent `save()` calls will write to the same
+            path.
+
+        """
+        self.path = path or self.path
+        with open(self.path) as f:
+            self._prev_dict = json.load(f)
+
+    def changed(self, key):
+        """Return true if the value for this key has changed since
+        the last save.
+
+        """
+        if self._prev_dict is None:
+            return True
+        return self.previous(key) != self.get(key)
+
+    def previous(self, key):
+        """Return previous value for this key, or None if there
+        is no "previous" value.
+
+        """
+        if self._prev_dict:
+            return self._prev_dict.get(key)
+        return None
+
+    def save(self):
+        """Save this config to disk.
+
+        Preserves items in _prev_dict that do not exist in self.
+
+        """
+        if self._prev_dict:
+            for k, v in self._prev_dict.iteritems():
+                if k not in self:
+                    self[k] = v
+        with open(self.path, 'w') as f:
+            json.dump(self, f)
+
+
 @cached
 def config(scope=None):
     """Juju charm configuration"""
@@ -163,7 +257,10 @@ def config(scope=None):
         config_cmd_line.append(scope)
     config_cmd_line.append('--format=json')
     try:
-        return json.loads(subprocess.check_output(config_cmd_line))
+        config_data = json.loads(subprocess.check_output(config_cmd_line))
+        if scope is not None:
+            return config_data
+        return Config(config_data)
     except ValueError:
         return None
 

hooks/charmhelpers/core/host.py

@@ -12,6 +12,7 @@ import random
 import string
 import subprocess
 import hashlib
+import apt_pkg
 
 from collections import OrderedDict
 
@@ -295,3 +296,16 @@ def get_nic_hwaddr(nic):
     if 'link/ether' in words:
         hwaddr = words[words.index('link/ether') + 1]
     return hwaddr
+
+
+def cmp_pkgrevno(package, revno, pkgcache=None):
+    '''Compare supplied revno with the revno of the installed package
+       1 => Installed revno is greater than supplied arg
+       0 => Installed revno is the same as supplied arg
+      -1 => Installed revno is less than supplied arg
+    '''
+    if not pkgcache:
+        apt_pkg.init()
+        pkgcache = apt_pkg.Cache()
+    pkg = pkgcache[package]
+    return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

hooks/charmhelpers/fetch/__init__.py

@@ -1,4 +1,5 @@
 import importlib
+import time
 from yaml import safe_load
 from charmhelpers.core.host import (
     lsb_release
@@ -15,6 +16,7 @@ from charmhelpers.core.hookenv import (
 import apt_pkg
 import os
 
+
 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
 """
@@ -56,10 +58,62 @@ CLOUD_ARCHIVE_POCKETS = {
     'precise-proposed/icehouse': 'precise-proposed/icehouse',
 }
 
+# The order of this list is very important. Handlers should be listed in from
+# least- to most-specific URL matching.
+FETCH_HANDLERS = (
+    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+)
+
+APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
+APT_NO_LOCK_RETRY_DELAY = 10  # Wait 10 seconds between apt lock checks.
+APT_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+
+
+class SourceConfigError(Exception):
+    pass
+
+
+class UnhandledSource(Exception):
+    pass
+
+
+class AptLockError(Exception):
+    pass
+
+
+class BaseFetchHandler(object):
+
+    """Base class for FetchHandler implementations in fetch plugins"""
+
+    def can_handle(self, source):
+        """Returns True if the source can be handled. Otherwise returns
+        a string explaining why it cannot"""
+        return "Wrong source type"
+
+    def install(self, source):
+        """Try to download and unpack the source. Return the path to the
+        unpacked files or raise UnhandledSource."""
+        raise UnhandledSource("Wrong source type {}".format(source))
+
+    def parse_url(self, url):
+        return urlparse(url)
+
+    def base_url(self, url):
+        """Return url without querystring or fragment"""
+        parts = list(self.parse_url(url))
+        parts[4:] = ['' for i in parts[4:]]
+        return urlunparse(parts)
+
 
 def filter_installed_packages(packages):
     """Returns a list of packages that require installation"""
     apt_pkg.init()
+
+    # Tell apt to build an in-memory cache to prevent race conditions (if
+    # another process is already building the cache).
+    apt_pkg.config.set("Dir::Cache::pkgcache", "")
+
     cache = apt_pkg.Cache()
     _pkgs = []
     for package in packages:
@@ -87,14 +141,7 @@ def apt_install(packages, options=None, fatal=False):
         cmd.extend(packages)
     log("Installing {} with options: {}".format(packages,
                                                 options))
-    env = os.environ.copy()
+    _run_apt_command(cmd, fatal)
-    if 'DEBIAN_FRONTEND' not in env:
-        env['DEBIAN_FRONTEND'] = 'noninteractive'
-
-    if fatal:
-        subprocess.check_call(cmd, env=env)
-    else:
-        subprocess.call(cmd, env=env)
 
 
 def apt_upgrade(options=None, fatal=False, dist=False):
@@ -109,24 +156,13 @@ def apt_upgrade(options=None, fatal=False, dist=False):
     else:
         cmd.append('upgrade')
     log("Upgrading with options: {}".format(options))
-
+    _run_apt_command(cmd, fatal)
-    env = os.environ.copy()
-    if 'DEBIAN_FRONTEND' not in env:
-        env['DEBIAN_FRONTEND'] = 'noninteractive'
-
-    if fatal:
-        subprocess.check_call(cmd, env=env)
-    else:
-        subprocess.call(cmd, env=env)
 
 
 def apt_update(fatal=False):
     """Update local apt cache"""
     cmd = ['apt-get', 'update']
-    if fatal:
+    _run_apt_command(cmd, fatal)
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
 
 
 def apt_purge(packages, fatal=False):
@@ -137,10 +173,7 @@ def apt_purge(packages, fatal=False):
     else:
         cmd.extend(packages)
     log("Purging {}".format(packages))
-    if fatal:
+    _run_apt_command(cmd, fatal)
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
 
 
 def apt_hold(packages, fatal=False):
@@ -151,6 +184,7 @@ def apt_hold(packages, fatal=False):
     else:
         cmd.extend(packages)
     log("Holding {}".format(packages))
+
     if fatal:
         subprocess.check_call(cmd)
     else:
@@ -188,10 +222,6 @@ def add_source(source, key=None):
                                key])
 
 
-class SourceConfigError(Exception):
-    pass
-
-
 def configure_sources(update=False,
                       sources_var='install_sources',
                       keys_var='install_keys'):
@@ -224,17 +254,6 @@ def configure_sources(update=False,
     if update:
         apt_update(fatal=True)
 
-# The order of this list is very important. Handlers should be listed in from
-# least- to most-specific URL matching.
-FETCH_HANDLERS = (
-    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
-    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
-)
-
-
-class UnhandledSource(Exception):
-    pass
-
 
 def install_remote(source):
     """
@@ -265,30 +284,6 @@ def install_from_config(config_var_name):
     return install_remote(source)
 
 
-class BaseFetchHandler(object):
-
-    """Base class for FetchHandler implementations in fetch plugins"""
-
-    def can_handle(self, source):
-        """Returns True if the source can be handled. Otherwise returns
-        a string explaining why it cannot"""
-        return "Wrong source type"
-
-    def install(self, source):
-        """Try to download and unpack the source. Return the path to the
-        unpacked files or raise UnhandledSource."""
-        raise UnhandledSource("Wrong source type {}".format(source))
-
-    def parse_url(self, url):
-        return urlparse(url)
-
-    def base_url(self, url):
-        """Return url without querystring or fragment"""
-        parts = list(self.parse_url(url))
-        parts[4:] = ['' for i in parts[4:]]
-        return urlunparse(parts)
-
-
 def plugins(fetch_handlers=None):
     if not fetch_handlers:
         fetch_handlers = FETCH_HANDLERS
@@ -306,3 +301,40 @@ def plugins(fetch_handlers=None):
             log("FetchHandler {} not found, skipping plugin".format(
                 handler_name))
     return plugin_list
+
+
+def _run_apt_command(cmd, fatal=False):
+    """
+    Run an APT command, checking output and retrying if the fatal flag is set
+    to True.
+
+    :param: cmd: str: The apt command to run.
+    :param: fatal: bool: Whether the command's output should be checked and
+        retried.
+    """
+    env = os.environ.copy()
+
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        retry_count = 0
+        result = None
+
+        # If the command is considered "fatal", we need to retry if the apt
+        # lock was not acquired.
+
+        while result is None or result == APT_NO_LOCK:
+            try:
+                result = subprocess.check_call(cmd, env=env)
+            except subprocess.CalledProcessError, e:
+                retry_count = retry_count + 1
+                if retry_count > APT_NO_LOCK_RETRY_COUNT:
+                    raise
+                result = e.returncode
+                log("Couldn't acquire DPKG lock. Will retry in {} seconds."
+                    "".format(APT_NO_LOCK_RETRY_DELAY))
+                time.sleep(APT_NO_LOCK_RETRY_DELAY)
+
+    else:
+        subprocess.call(cmd, env=env)

hooks/keystone_context.py

@@ -31,7 +31,7 @@ class ApacheSSLContext(context.ApacheSSLContext):
         return super(ApacheSSLContext, self).__call__()
 
     def configure_cert(self):
-        #import keystone_ssl as ssl
+        # import keystone_ssl as ssl
         from keystone_utils import SSH_USER, get_ca
         if not os.path.isdir('/etc/apache2/ssl'):
             os.mkdir('/etc/apache2/ssl')

hooks/keystone_utils.py

@@ -168,6 +168,10 @@ valid_services = {
     "heat-cfn": {
         "type": "cloudformation",
         "desc": "Heat CloudFormation API"
+    },
+    "image-stream": {
+        "type": "product-streams",
+        "desc": "Ubuntu Product Streams"
     }
 }