From 6cddd9e0c79d02148c50a9484f2e329b0816bf49 Mon Sep 17 00:00:00 2001 From: Adam Gandelman Date: Tue, 5 Nov 2013 19:50:44 -0800 Subject: [PATCH] Update and sync helpers. --- charm-helpers.yaml | 3 - hooks/charmhelpers/contrib/openstack/utils.py | 83 +++++++++++++++++-- .../contrib/storage/linux/ceph.py | 30 ++++++- hooks/charmhelpers/core/hookenv.py | 20 +++++ hooks/charmhelpers/fetch/__init__.py | 34 +++++++- hooks/charmhelpers/fetch/bzrurl.py | 2 +- 6 files changed, 157 insertions(+), 15 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 47df1e89..ac41438a 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,6 +1,3 @@ -# NOTE: Merge conflicts. -#branch: lp:~james-page/charm-helpers/nvp-updates -#branch: lp:~openstack-charmers/charm-helpers/os-alternatives branch: lp:charm-helpers destination: hooks/charmhelpers include: diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 62d207f9..d66afd74 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -13,19 +13,28 @@ from charmhelpers.core.hookenv import ( config, log as juju_log, charm_dir, + ERROR, + INFO ) -from charmhelpers.core.host import ( - lsb_release, +from charmhelpers.contrib.storage.linux.lvm import ( + deactivate_lvm_volume_group, + is_lvm_physical_volume, + remove_lvm_physical_volume, ) -from charmhelpers.fetch import ( - apt_install, -) +from charmhelpers.core.host import lsb_release, mounts, umount +from charmhelpers.fetch import apt_install +from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk +from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' +DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed ' + 'restricted main multiverse universe') + + UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('oneiric', 'diablo'), ('precise', 'essex'), @@ -57,6 +66,8 @@ SWIFT_CODENAMES = OrderedDict([ ('1.9.0', 'havana'), ]) +DEFAULT_LOOPBACK_SIZE = '5G' + def error_out(msg): juju_log("FATAL ERROR: %s" % msg, level='ERROR') @@ -67,7 +78,7 @@ def get_os_codename_install_source(src): '''Derive OpenStack release codename from a given installation source.''' ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] rel = '' - if src == 'distro': + if src in ['distro', 'distro-proposed']: try: rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel] except KeyError: @@ -202,6 +213,10 @@ def configure_installation_source(rel): '''Configure apt installation source.''' if rel == 'distro': return + elif rel == 'distro-proposed': + ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] + with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: + f.write(DISTRO_PROPOSED % ubuntu_rel) elif rel[:4] == "ppa:": src = rel subprocess.check_call(["add-apt-repository", "-y", src]) @@ -299,6 +314,62 @@ def openstack_upgrade_available(package): return apt.version_compare(available_vers, cur_vers) == 1 +def ensure_block_device(block_device): + ''' + Confirm block_device, create as loopback if necessary. + + :param block_device: str: Full path of block device to ensure. + + :returns: str: Full path of ensured block device. + ''' + _none = ['None', 'none', None] + if (block_device in _none): + error_out('prepare_storage(): Missing required input: ' + 'block_device=%s.' % block_device, level=ERROR) + + if block_device.startswith('/dev/'): + bdev = block_device + elif block_device.startswith('/'): + _bd = block_device.split('|') + if len(_bd) == 2: + bdev, size = _bd + else: + bdev = block_device + size = DEFAULT_LOOPBACK_SIZE + bdev = ensure_loopback_device(bdev, size) + else: + bdev = '/dev/%s' % block_device + + if not is_block_device(bdev): + error_out('Failed to locate valid block device at %s' % bdev, + level=ERROR) + + return bdev + + +def clean_storage(block_device): + ''' + Ensures a block device is clean. That is: + - unmounted + - any lvm volume groups are deactivated + - any lvm physical device signatures removed + - partition table wiped + + :param block_device: str: Full path to block device to clean. + ''' + for mp, d in mounts(): + if d == block_device: + juju_log('clean_storage(): %s is mounted @ %s, unmounting.' % + (d, mp), level=INFO) + umount(mp, persist=True) + + if is_lvm_physical_volume(block_device): + deactivate_lvm_volume_group(block_device) + remove_lvm_physical_volume(block_device) + else: + zap_disk(block_device) + + def is_ip(address): """ Returns True if address is a valid IP address. diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 9bb9530c..69b879ca 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -102,8 +102,12 @@ def get_osds(service): Return a list of all Ceph Object Storage Daemons currently in the cluster ''' - return json.loads(check_output(['ceph', '--id', service, - 'osd', 'ls', '--format=json'])) + version = ceph_version() + if version and version >= '0.56': + return json.loads(check_output(['ceph', '--id', service, + 'osd', 'ls', '--format=json'])) + else: + return None def create_pool(service, name, replicas=2): @@ -114,7 +118,13 @@ def create_pool(service, name, replicas=2): return # Calculate the number of placement groups based # on upstream recommended best practices. - pgnum = (len(get_osds(service)) * 100 / replicas) + osds = get_osds(service) + if osds: + pgnum = (len(osds) * 100 / replicas) + else: + # NOTE(james-page): Default to 200 for older ceph versions + # which don't support OSD query from cli + pgnum = 200 cmd = [ 'ceph', '--id', service, 'osd', 'pool', 'create', @@ -357,3 +367,17 @@ def ensure_ceph_keyring(service, user=None, group=None): if user and group: check_call(['chown', '%s.%s' % (user, group), keyring]) return True + + +def ceph_version(): + ''' Retrieve the local version of ceph ''' + if os.path.exists('/usr/bin/ceph'): + cmd = ['ceph', '-v'] + output = check_output(cmd) + output = output.split() + if len(output) > 3: + return output[2] + else: + return None + else: + return None diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index e86f8654..bb196dfa 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -285,6 +285,26 @@ def relations(): return rels +@cached +def is_relation_made(relation, keys='private-address'): + ''' + Determine whether a relation is established by checking for + presence of key(s). If a list of keys is provided, they + must all be present for the relation to be identified as made + ''' + if isinstance(keys, str): + keys = [keys] + for r_id in relation_ids(relation): + for unit in related_units(r_id): + context = {} + for k in keys: + context[k] = relation_get(k, rid=r_id, + unit=unit) + if None not in context.values(): + return True + return False + + def open_port(port, protocol="TCP"): """Open a service network port""" _args = ['open-port'] diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 68f55f00..fa0172a9 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -20,6 +20,32 @@ deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main PROPOSED_POCKET = """# Proposed deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted """ +CLOUD_ARCHIVE_POCKETS = { + # Folsom + 'folsom': 'precise-updates/folsom', + 'precise-folsom': 'precise-updates/folsom', + 'precise-folsom/updates': 'precise-updates/folsom', + 'precise-updates/folsom': 'precise-updates/folsom', + 'folsom/proposed': 'precise-proposed/folsom', + 'precise-folsom/proposed': 'precise-proposed/folsom', + 'precise-proposed/folsom': 'precise-proposed/folsom', + # Grizzly + 'grizzly': 'precise-updates/grizzly', + 'precise-grizzly': 'precise-updates/grizzly', + 'precise-grizzly/updates': 'precise-updates/grizzly', + 'precise-updates/grizzly': 'precise-updates/grizzly', + 'grizzly/proposed': 'precise-proposed/grizzly', + 'precise-grizzly/proposed': 'precise-proposed/grizzly', + 'precise-proposed/grizzly': 'precise-proposed/grizzly', + # Havana + 'havana': 'precise-updates/havana', + 'precise-havana': 'precise-updates/havana', + 'precise-havana/updates': 'precise-updates/havana', + 'precise-updates/havana': 'precise-updates/havana', + 'havana/proposed': 'precise-proposed/havana', + 'precies-havana/proposed': 'precise-proposed/havana', + 'precise-proposed/havana': 'precise-proposed/havana', +} def filter_installed_packages(packages): @@ -96,14 +122,18 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): if (source.startswith('ppa:') or source.startswith('http:') or - source.startswith('deb ')): + source.startswith('deb ') or + source.startswith('cloud-archive:')): subprocess.check_call(['add-apt-repository', '--yes', source]) elif source.startswith('cloud:'): apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), fatal=True) pocket = source.split(':')[-1] + if pocket not in CLOUD_ARCHIVE_POCKETS: + raise SourceConfigError('Unsupported cloud: source option %s' % pocket) + actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: - apt.write(CLOUD_ARCHIVE.format(pocket)) + apt.write(CLOUD_ARCHIVE.format(actual_pocket)) elif source == 'proposed': release = lsb_release()['DISTRIB_CODENAME'] with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index c348b4bb..db5dd9a3 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -12,6 +12,7 @@ except ImportError: apt_install("python-bzrlib") from bzrlib.branch import Branch + class BzrUrlFetchHandler(BaseFetchHandler): """Handler for bazaar branches via generic and lp URLs""" def can_handle(self, source): @@ -46,4 +47,3 @@ class BzrUrlFetchHandler(BaseFetchHandler): except OSError as e: raise UnhandledSource(e.strerror) return dest_dir -