Update and sync helpers.

charm-helpers.yaml

@@ -1,6 +1,3 @@
-# NOTE: Merge conflicts.
-#branch: lp:~james-page/charm-helpers/nvp-updates
-#branch: lp:~openstack-charmers/charm-helpers/os-alternatives
 branch: lp:charm-helpers
 destination: hooks/charmhelpers
 include:

hooks/charmhelpers/contrib/openstack/utils.py

@@ -13,19 +13,28 @@ from charmhelpers.core.hookenv import (
     config,
     log as juju_log,
     charm_dir,
+    ERROR,
+    INFO
 )
 
-from charmhelpers.core.host import (
-    lsb_release,
+from charmhelpers.contrib.storage.linux.lvm import (
+    deactivate_lvm_volume_group,
+    is_lvm_physical_volume,
+    remove_lvm_physical_volume,
 )
 
-from charmhelpers.fetch import (
-    apt_install,
-)
+from charmhelpers.core.host import lsb_release, mounts, umount
+from charmhelpers.fetch import apt_install
+from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
+from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
 CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
 CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
 
+DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
+                   'restricted main multiverse universe')
+
+
 UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('oneiric', 'diablo'),
     ('precise', 'essex'),
@@ -57,6 +66,8 @@ SWIFT_CODENAMES = OrderedDict([
     ('1.9.0', 'havana'),
 ])
 
+DEFAULT_LOOPBACK_SIZE = '5G'
+
 
 def error_out(msg):
     juju_log("FATAL ERROR: %s" % msg, level='ERROR')
@@ -67,7 +78,7 @@ def get_os_codename_install_source(src):
     '''Derive OpenStack release codename from a given installation source.'''
     ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
     rel = ''
-    if src == 'distro':
+    if src in ['distro', 'distro-proposed']:
         try:
             rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel]
         except KeyError:
@@ -202,6 +213,10 @@ def configure_installation_source(rel):
     '''Configure apt installation source.'''
     if rel == 'distro':
         return
+    elif rel == 'distro-proposed':
+        ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
+        with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
+            f.write(DISTRO_PROPOSED % ubuntu_rel)
     elif rel[:4] == "ppa:":
         src = rel
         subprocess.check_call(["add-apt-repository", "-y", src])
@@ -299,6 +314,62 @@ def openstack_upgrade_available(package):
     return apt.version_compare(available_vers, cur_vers) == 1
 
 
+def ensure_block_device(block_device):
+    '''
+    Confirm block_device, create as loopback if necessary.
+
+    :param block_device: str: Full path of block device to ensure.
+
+    :returns: str: Full path of ensured block device.
+    '''
+    _none = ['None', 'none', None]
+    if (block_device in _none):
+        error_out('prepare_storage(): Missing required input: '
+                  'block_device=%s.' % block_device, level=ERROR)
+
+    if block_device.startswith('/dev/'):
+        bdev = block_device
+    elif block_device.startswith('/'):
+        _bd = block_device.split('|')
+        if len(_bd) == 2:
+            bdev, size = _bd
+        else:
+            bdev = block_device
+            size = DEFAULT_LOOPBACK_SIZE
+        bdev = ensure_loopback_device(bdev, size)
+    else:
+        bdev = '/dev/%s' % block_device
+
+    if not is_block_device(bdev):
+        error_out('Failed to locate valid block device at %s' % bdev,
+                  level=ERROR)
+
+    return bdev
+
+
+def clean_storage(block_device):
+    '''
+    Ensures a block device is clean.  That is:
+        - unmounted
+        - any lvm volume groups are deactivated
+        - any lvm physical device signatures removed
+        - partition table wiped
+
+    :param block_device: str: Full path to block device to clean.
+    '''
+    for mp, d in mounts():
+        if d == block_device:
+            juju_log('clean_storage(): %s is mounted @ %s, unmounting.' %
+                     (d, mp), level=INFO)
+            umount(mp, persist=True)
+
+    if is_lvm_physical_volume(block_device):
+        deactivate_lvm_volume_group(block_device)
+        remove_lvm_physical_volume(block_device)
+    else:
+        zap_disk(block_device)
+
+
 def is_ip(address):
     """
     Returns True if address is a valid IP address.

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -102,8 +102,12 @@ def get_osds(service):
     Return a list of all Ceph Object Storage Daemons
     currently in the cluster
     '''
-    return json.loads(check_output(['ceph', '--id', service,
-                                    'osd', 'ls', '--format=json']))
+    version = ceph_version()
+    if version and version >= '0.56':
+        return json.loads(check_output(['ceph', '--id', service,
+                                        'osd', 'ls', '--format=json']))
+    else:
+        return None
 
 
 def create_pool(service, name, replicas=2):
@@ -114,7 +118,13 @@ def create_pool(service, name, replicas=2):
         return
     # Calculate the number of placement groups based
     # on upstream recommended best practices.
-    pgnum = (len(get_osds(service)) * 100 / replicas)
+    osds = get_osds(service)
+    if osds:
+        pgnum = (len(osds) * 100 / replicas)
+    else:
+        # NOTE(james-page): Default to 200 for older ceph versions
+        # which don't support OSD query from cli
+        pgnum = 200
     cmd = [
         'ceph', '--id', service,
         'osd', 'pool', 'create',
@@ -357,3 +367,17 @@ def ensure_ceph_keyring(service, user=None, group=None):
     if user and group:
         check_call(['chown', '%s.%s' % (user, group), keyring])
     return True
+
+
+def ceph_version():
+    ''' Retrieve the local version of ceph '''
+    if os.path.exists('/usr/bin/ceph'):
+        cmd = ['ceph', '-v']
+        output = check_output(cmd)
+        output = output.split()
+        if len(output) > 3:
+            return output[2]
+        else:
+            return None
+    else:
+        return None

hooks/charmhelpers/core/hookenv.py

@@ -285,6 +285,26 @@ def relations():
     return rels
 
 
+@cached
+def is_relation_made(relation, keys='private-address'):
+    '''
+    Determine whether a relation is established by checking for
+    presence of key(s).  If a list of keys is provided, they
+    must all be present for the relation to be identified as made
+    '''
+    if isinstance(keys, str):
+        keys = [keys]
+    for r_id in relation_ids(relation):
+        for unit in related_units(r_id):
+            context = {}
+            for k in keys:
+                context[k] = relation_get(k, rid=r_id,
+                                          unit=unit)
+            if None not in context.values():
+                return True
+    return False
+
+
 def open_port(port, protocol="TCP"):
     """Open a service network port"""
     _args = ['open-port']

hooks/charmhelpers/fetch/__init__.py

@@ -20,6 +20,32 @@ deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
 PROPOSED_POCKET = """# Proposed
 deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
 """
+CLOUD_ARCHIVE_POCKETS = {
+    # Folsom
+    'folsom': 'precise-updates/folsom',
+    'precise-folsom': 'precise-updates/folsom',
+    'precise-folsom/updates': 'precise-updates/folsom',
+    'precise-updates/folsom': 'precise-updates/folsom',
+    'folsom/proposed': 'precise-proposed/folsom',
+    'precise-folsom/proposed': 'precise-proposed/folsom',
+    'precise-proposed/folsom': 'precise-proposed/folsom',
+    # Grizzly
+    'grizzly': 'precise-updates/grizzly',
+    'precise-grizzly': 'precise-updates/grizzly',
+    'precise-grizzly/updates': 'precise-updates/grizzly',
+    'precise-updates/grizzly': 'precise-updates/grizzly',
+    'grizzly/proposed': 'precise-proposed/grizzly',
+    'precise-grizzly/proposed': 'precise-proposed/grizzly',
+    'precise-proposed/grizzly': 'precise-proposed/grizzly',
+    # Havana
+    'havana': 'precise-updates/havana',
+    'precise-havana': 'precise-updates/havana',
+    'precise-havana/updates': 'precise-updates/havana',
+    'precise-updates/havana': 'precise-updates/havana',
+    'havana/proposed': 'precise-proposed/havana',
+    'precies-havana/proposed': 'precise-proposed/havana',
+    'precise-proposed/havana': 'precise-proposed/havana',
+}
 
 
 def filter_installed_packages(packages):
@@ -96,14 +122,18 @@ def apt_hold(packages, fatal=False):
 def add_source(source, key=None):
     if (source.startswith('ppa:') or
         source.startswith('http:') or
-        source.startswith('deb ')):
+        source.startswith('deb ') or
+        source.startswith('cloud-archive:')):
         subprocess.check_call(['add-apt-repository', '--yes', source])
     elif source.startswith('cloud:'):
         apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
                     fatal=True)
         pocket = source.split(':')[-1]
+        if pocket not in CLOUD_ARCHIVE_POCKETS:
+            raise SourceConfigError('Unsupported cloud: source option %s' % pocket)
+        actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
         with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
-            apt.write(CLOUD_ARCHIVE.format(pocket))
+            apt.write(CLOUD_ARCHIVE.format(actual_pocket))
     elif source == 'proposed':
         release = lsb_release()['DISTRIB_CODENAME']
         with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:

hooks/charmhelpers/fetch/bzrurl.py

@@ -12,6 +12,7 @@ except ImportError:
     apt_install("python-bzrlib")
     from bzrlib.branch import Branch
 
+
 class BzrUrlFetchHandler(BaseFetchHandler):
     """Handler for bazaar branches via generic and lp URLs"""
     def can_handle(self, source):
@@ -46,4 +47,3 @@ class BzrUrlFetchHandler(BaseFetchHandler):
         except OSError as e:
             raise UnhandledSource(e.strerror)
         return dest_dir
-