diff --git a/Makefile b/Makefile index 18a2771c..11aa3028 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,6 @@ lint: sync: @charm-helper-sync -c charm-helpers-sync.yaml -publish: +publish: lint bzr push lp:charms/ceph-osd bzr push lp:charms/trusty/ceph-osd diff --git a/hooks/ceph.py b/hooks/ceph.py index fca22549..21b919df 100644 --- a/hooks/ceph.py +++ b/hooks/ceph.py @@ -23,6 +23,7 @@ from charmhelpers.core.hookenv import ( from charmhelpers.contrib.storage.linux.utils import ( zap_disk, is_block_device, + is_device_mounted, ) from utils import ( get_unit_hostname, @@ -340,7 +341,7 @@ def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False): log('Looks like {} is already an OSD, skipping.'.format(dev)) return - if device_mounted(dev): + if is_device_mounted(dev): log('Looks like {} is in use, skipping.'.format(dev)) return @@ -383,9 +384,5 @@ def osdize_dir(path): subprocess.check_call(cmd) -def device_mounted(dev): - return subprocess.call(['grep', '-wqs', dev + '1', '/proc/mounts']) == 0 - - def filesystem_mounted(fs): return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0 diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index eed99ae3..b87ef26d 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -1,4 +1,5 @@ -from os import stat +import os +import re from stat import S_ISBLK from subprocess import ( @@ -14,7 +15,9 @@ def is_block_device(path): :returns: boolean: True if path is a block device, False if not. ''' - return S_ISBLK(stat(path).st_mode) + if not os.path.exists(path): + return False + return S_ISBLK(os.stat(path).st_mode) def zap_disk(block_device): @@ -29,7 +32,18 @@ def zap_disk(block_device): '--clear', block_device]) dev_end = check_output(['blockdev', '--getsz', block_device]) gpt_end = int(dev_end.split()[0]) - 100 - check_call(['dd', 'if=/dev/zero', 'of=%s'%(block_device), + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), 'bs=1M', 'count=1']) - check_call(['dd', 'if=/dev/zero', 'of=%s'%(block_device), - 'bs=512', 'count=100', 'seek=%s'%(gpt_end)]) + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), + 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) + +def is_device_mounted(device): + '''Given a device path, return True if that device is mounted, and False + if it isn't. + + :param device: str: Full path of the device to check. + :returns: boolean: True if the path represents a mounted device, False if + it doesn't. + ''' + out = check_output(['mount']) + return bool(re.search(device + r"[0-9]+\b", out)) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 505c202d..c2e66f66 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -155,6 +155,100 @@ def hook_name(): return os.path.basename(sys.argv[0]) +class Config(dict): + """A Juju charm config dictionary that can write itself to + disk (as json) and track which values have changed since + the previous hook invocation. + + Do not instantiate this object directly - instead call + ``hookenv.config()`` + + Example usage:: + + >>> # inside a hook + >>> from charmhelpers.core import hookenv + >>> config = hookenv.config() + >>> config['foo'] + 'bar' + >>> config['mykey'] = 'myval' + >>> config.save() + + + >>> # user runs `juju set mycharm foo=baz` + >>> # now we're inside subsequent config-changed hook + >>> config = hookenv.config() + >>> config['foo'] + 'baz' + >>> # test to see if this val has changed since last hook + >>> config.changed('foo') + True + >>> # what was the previous value? + >>> config.previous('foo') + 'bar' + >>> # keys/values that we add are preserved across hooks + >>> config['mykey'] + 'myval' + >>> # don't forget to save at the end of hook! + >>> config.save() + + """ + CONFIG_FILE_NAME = '.juju-persistent-config' + + def __init__(self, *args, **kw): + super(Config, self).__init__(*args, **kw) + self._prev_dict = None + self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) + if os.path.exists(self.path): + self.load_previous() + + def load_previous(self, path=None): + """Load previous copy of config from disk so that current values + can be compared to previous values. + + :param path: + + File path from which to load the previous config. If `None`, + config is loaded from the default location. If `path` is + specified, subsequent `save()` calls will write to the same + path. + + """ + self.path = path or self.path + with open(self.path) as f: + self._prev_dict = json.load(f) + + def changed(self, key): + """Return true if the value for this key has changed since + the last save. + + """ + if self._prev_dict is None: + return True + return self.previous(key) != self.get(key) + + def previous(self, key): + """Return previous value for this key, or None if there + is no "previous" value. + + """ + if self._prev_dict: + return self._prev_dict.get(key) + return None + + def save(self): + """Save this config to disk. + + Preserves items in _prev_dict that do not exist in self. + + """ + if self._prev_dict: + for k, v in self._prev_dict.iteritems(): + if k not in self: + self[k] = v + with open(self.path, 'w') as f: + json.dump(self, f) + + @cached def config(scope=None): """Juju charm configuration""" @@ -163,7 +257,10 @@ def config(scope=None): config_cmd_line.append(scope) config_cmd_line.append('--format=json') try: - return json.loads(subprocess.check_output(config_cmd_line)) + config_data = json.loads(subprocess.check_output(config_cmd_line)) + if scope is not None: + return config_data + return Config(config_data) except ValueError: return None diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cfd26847..186147f6 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -12,6 +12,7 @@ import random import string import subprocess import hashlib +import apt_pkg from collections import OrderedDict @@ -295,3 +296,16 @@ def get_nic_hwaddr(nic): if 'link/ether' in words: hwaddr = words[words.index('link/ether') + 1] return hwaddr + + +def cmp_pkgrevno(package, revno, pkgcache=None): + '''Compare supplied revno with the revno of the installed package + 1 => Installed revno is greater than supplied arg + 0 => Installed revno is the same as supplied arg + -1 => Installed revno is less than supplied arg + ''' + if not pkgcache: + apt_pkg.init() + pkgcache = apt_pkg.Cache() + pkg = pkgcache[package] + return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index dce7db4c..e1e17dae 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -1,4 +1,5 @@ import importlib +import time from yaml import safe_load from charmhelpers.core.host import ( lsb_release @@ -15,6 +16,7 @@ from charmhelpers.core.hookenv import ( import apt_pkg import os + CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main """ @@ -56,10 +58,62 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/icehouse': 'precise-proposed/icehouse', } +# The order of this list is very important. Handlers should be listed in from +# least- to most-specific URL matching. +FETCH_HANDLERS = ( + 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', + 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', +) + +APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. +APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. +APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. + + +class SourceConfigError(Exception): + pass + + +class UnhandledSource(Exception): + pass + + +class AptLockError(Exception): + pass + + +class BaseFetchHandler(object): + + """Base class for FetchHandler implementations in fetch plugins""" + + def can_handle(self, source): + """Returns True if the source can be handled. Otherwise returns + a string explaining why it cannot""" + return "Wrong source type" + + def install(self, source): + """Try to download and unpack the source. Return the path to the + unpacked files or raise UnhandledSource.""" + raise UnhandledSource("Wrong source type {}".format(source)) + + def parse_url(self, url): + return urlparse(url) + + def base_url(self, url): + """Return url without querystring or fragment""" + parts = list(self.parse_url(url)) + parts[4:] = ['' for i in parts[4:]] + return urlunparse(parts) + def filter_installed_packages(packages): """Returns a list of packages that require installation""" apt_pkg.init() + + # Tell apt to build an in-memory cache to prevent race conditions (if + # another process is already building the cache). + apt_pkg.config.set("Dir::Cache::pkgcache", "") + cache = apt_pkg.Cache() _pkgs = [] for package in packages: @@ -87,14 +141,7 @@ def apt_install(packages, options=None, fatal=False): cmd.extend(packages) log("Installing {} with options: {}".format(packages, options)) - env = os.environ.copy() - if 'DEBIAN_FRONTEND' not in env: - env['DEBIAN_FRONTEND'] = 'noninteractive' - - if fatal: - subprocess.check_call(cmd, env=env) - else: - subprocess.call(cmd, env=env) + _run_apt_command(cmd, fatal) def apt_upgrade(options=None, fatal=False, dist=False): @@ -109,24 +156,13 @@ def apt_upgrade(options=None, fatal=False, dist=False): else: cmd.append('upgrade') log("Upgrading with options: {}".format(options)) - - env = os.environ.copy() - if 'DEBIAN_FRONTEND' not in env: - env['DEBIAN_FRONTEND'] = 'noninteractive' - - if fatal: - subprocess.check_call(cmd, env=env) - else: - subprocess.call(cmd, env=env) + _run_apt_command(cmd, fatal) def apt_update(fatal=False): """Update local apt cache""" cmd = ['apt-get', 'update'] - if fatal: - subprocess.check_call(cmd) - else: - subprocess.call(cmd) + _run_apt_command(cmd, fatal) def apt_purge(packages, fatal=False): @@ -137,10 +173,7 @@ def apt_purge(packages, fatal=False): else: cmd.extend(packages) log("Purging {}".format(packages)) - if fatal: - subprocess.check_call(cmd) - else: - subprocess.call(cmd) + _run_apt_command(cmd, fatal) def apt_hold(packages, fatal=False): @@ -151,6 +184,7 @@ def apt_hold(packages, fatal=False): else: cmd.extend(packages) log("Holding {}".format(packages)) + if fatal: subprocess.check_call(cmd) else: @@ -188,10 +222,6 @@ def add_source(source, key=None): key]) -class SourceConfigError(Exception): - pass - - def configure_sources(update=False, sources_var='install_sources', keys_var='install_keys'): @@ -224,17 +254,6 @@ def configure_sources(update=False, if update: apt_update(fatal=True) -# The order of this list is very important. Handlers should be listed in from -# least- to most-specific URL matching. -FETCH_HANDLERS = ( - 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', - 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', -) - - -class UnhandledSource(Exception): - pass - def install_remote(source): """ @@ -265,30 +284,6 @@ def install_from_config(config_var_name): return install_remote(source) -class BaseFetchHandler(object): - - """Base class for FetchHandler implementations in fetch plugins""" - - def can_handle(self, source): - """Returns True if the source can be handled. Otherwise returns - a string explaining why it cannot""" - return "Wrong source type" - - def install(self, source): - """Try to download and unpack the source. Return the path to the - unpacked files or raise UnhandledSource.""" - raise UnhandledSource("Wrong source type {}".format(source)) - - def parse_url(self, url): - return urlparse(url) - - def base_url(self, url): - """Return url without querystring or fragment""" - parts = list(self.parse_url(url)) - parts[4:] = ['' for i in parts[4:]] - return urlunparse(parts) - - def plugins(fetch_handlers=None): if not fetch_handlers: fetch_handlers = FETCH_HANDLERS @@ -306,3 +301,40 @@ def plugins(fetch_handlers=None): log("FetchHandler {} not found, skipping plugin".format( handler_name)) return plugin_list + + +def _run_apt_command(cmd, fatal=False): + """ + Run an APT command, checking output and retrying if the fatal flag is set + to True. + + :param: cmd: str: The apt command to run. + :param: fatal: bool: Whether the command's output should be checked and + retried. + """ + env = os.environ.copy() + + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + + if fatal: + retry_count = 0 + result = None + + # If the command is considered "fatal", we need to retry if the apt + # lock was not acquired. + + while result is None or result == APT_NO_LOCK: + try: + result = subprocess.check_call(cmd, env=env) + except subprocess.CalledProcessError, e: + retry_count = retry_count + 1 + if retry_count > APT_NO_LOCK_RETRY_COUNT: + raise + result = e.returncode + log("Couldn't acquire DPKG lock. Will retry in {} seconds." + "".format(APT_NO_LOCK_RETRY_DELAY)) + time.sleep(APT_NO_LOCK_RETRY_DELAY) + + else: + subprocess.call(cmd, env=env)