From 6e8078e6c518345107eb31bb326696a32f2be17a Mon Sep 17 00:00:00 2001 From: Christopher Glass Date: Mon, 19 May 2014 13:42:30 +0200 Subject: [PATCH 1/3] Updating charm-helpers from lp:charm-helpers revision 153 --- .../charmhelpers/contrib/openstack/context.py | 2 +- .../charmhelpers/contrib/openstack/neutron.py | 18 +- hooks/charmhelpers/contrib/openstack/utils.py | 9 +- .../charmhelpers/contrib/storage/linux/lvm.py | 2 +- .../contrib/storage/linux/utils.py | 33 +++- hooks/charmhelpers/core/hookenv.py | 99 ++++++++++- hooks/charmhelpers/core/host.py | 14 ++ hooks/charmhelpers/fetch/__init__.py | 160 +++++++++++------- 8 files changed, 263 insertions(+), 74 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index e92b1cc8..474d51ea 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -570,7 +570,7 @@ class NeutronContext(OSContextGenerator): if self.plugin == 'ovs': ctxt.update(self.ovs_ctxt()) - elif self.plugin == 'nvp': + elif self.plugin in ['nvp', 'nsx']: ctxt.update(self.nvp_ctxt()) alchemy_flags = config('neutron-alchemy-flags') diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 563e804d..ba97622c 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -114,14 +114,30 @@ def neutron_plugins(): 'server_packages': ['neutron-server', 'neutron-plugin-nicira'], 'server_services': ['neutron-server'] + }, + 'nsx': { + 'config': '/etc/neutron/plugins/vmware/nsx.ini', + 'driver': 'vmware', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', + 'neutron-plugin-vmware'], + 'server_services': ['neutron-server'] } } - # NOTE: patch in ml2 plugin for icehouse onwards if release >= 'icehouse': + # NOTE: patch in ml2 plugin for icehouse onwards plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini' plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin' plugins['ovs']['server_packages'] = ['neutron-server', 'neutron-plugin-ml2'] + # NOTE: patch in vmware renames nvp->nsx for icehouse onwards + plugins['nvp'] = plugins['nsx'] return plugins diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ac261fd7..cb69ca26 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -131,6 +131,11 @@ def get_os_version_codename(codename): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' apt.init() + + # Tell apt to build an in-memory cache to prevent race conditions (if + # another process is already building the cache). + apt.config.set("Dir::Cache::pkgcache", "") + cache = apt.Cache() try: @@ -183,7 +188,7 @@ def get_os_version_package(pkg, fatal=True): if cname == codename: return version #e = "Could not determine OpenStack version for package: %s" % pkg - #error_out(e) + # error_out(e) os_rel = None @@ -401,6 +406,8 @@ def ns_query(address): rtype = 'PTR' elif isinstance(address, basestring): rtype = 'A' + else: + return None answers = dns.resolver.query(address, rtype) if answers: diff --git a/hooks/charmhelpers/contrib/storage/linux/lvm.py b/hooks/charmhelpers/contrib/storage/linux/lvm.py index 6e29181a..8ac7fecc 100644 --- a/hooks/charmhelpers/contrib/storage/linux/lvm.py +++ b/hooks/charmhelpers/contrib/storage/linux/lvm.py @@ -62,7 +62,7 @@ def list_lvm_volume_group(block_device): pvd = check_output(['pvdisplay', block_device]).splitlines() for l in pvd: if l.strip().startswith('VG Name'): - vg = ' '.join(l.split()).split(' ').pop() + vg = ' '.join(l.strip().split()[2:]) return vg diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index 5349c3ea..b87ef26d 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -1,8 +1,11 @@ -from os import stat +import os +import re from stat import S_ISBLK from subprocess import ( - check_call + check_call, + check_output, + call ) @@ -12,7 +15,9 @@ def is_block_device(path): :returns: boolean: True if path is a block device, False if not. ''' - return S_ISBLK(stat(path).st_mode) + if not os.path.exists(path): + return False + return S_ISBLK(os.stat(path).st_mode) def zap_disk(block_device): @@ -22,5 +27,23 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', '--clear', - '--mbrtogpt', block_device]) + # sometimes sgdisk exits non-zero; this is OK, dd will clean up + call(['sgdisk', '--zap-all', '--mbrtogpt', + '--clear', block_device]) + dev_end = check_output(['blockdev', '--getsz', block_device]) + gpt_end = int(dev_end.split()[0]) - 100 + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), + 'bs=1M', 'count=1']) + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), + 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) + +def is_device_mounted(device): + '''Given a device path, return True if that device is mounted, and False + if it isn't. + + :param device: str: Full path of the device to check. + :returns: boolean: True if the path represents a mounted device, False if + it doesn't. + ''' + out = check_output(['mount']) + return bool(re.search(device + r"[0-9]+\b", out)) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 505c202d..c2e66f66 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -155,6 +155,100 @@ def hook_name(): return os.path.basename(sys.argv[0]) +class Config(dict): + """A Juju charm config dictionary that can write itself to + disk (as json) and track which values have changed since + the previous hook invocation. + + Do not instantiate this object directly - instead call + ``hookenv.config()`` + + Example usage:: + + >>> # inside a hook + >>> from charmhelpers.core import hookenv + >>> config = hookenv.config() + >>> config['foo'] + 'bar' + >>> config['mykey'] = 'myval' + >>> config.save() + + + >>> # user runs `juju set mycharm foo=baz` + >>> # now we're inside subsequent config-changed hook + >>> config = hookenv.config() + >>> config['foo'] + 'baz' + >>> # test to see if this val has changed since last hook + >>> config.changed('foo') + True + >>> # what was the previous value? + >>> config.previous('foo') + 'bar' + >>> # keys/values that we add are preserved across hooks + >>> config['mykey'] + 'myval' + >>> # don't forget to save at the end of hook! + >>> config.save() + + """ + CONFIG_FILE_NAME = '.juju-persistent-config' + + def __init__(self, *args, **kw): + super(Config, self).__init__(*args, **kw) + self._prev_dict = None + self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) + if os.path.exists(self.path): + self.load_previous() + + def load_previous(self, path=None): + """Load previous copy of config from disk so that current values + can be compared to previous values. + + :param path: + + File path from which to load the previous config. If `None`, + config is loaded from the default location. If `path` is + specified, subsequent `save()` calls will write to the same + path. + + """ + self.path = path or self.path + with open(self.path) as f: + self._prev_dict = json.load(f) + + def changed(self, key): + """Return true if the value for this key has changed since + the last save. + + """ + if self._prev_dict is None: + return True + return self.previous(key) != self.get(key) + + def previous(self, key): + """Return previous value for this key, or None if there + is no "previous" value. + + """ + if self._prev_dict: + return self._prev_dict.get(key) + return None + + def save(self): + """Save this config to disk. + + Preserves items in _prev_dict that do not exist in self. + + """ + if self._prev_dict: + for k, v in self._prev_dict.iteritems(): + if k not in self: + self[k] = v + with open(self.path, 'w') as f: + json.dump(self, f) + + @cached def config(scope=None): """Juju charm configuration""" @@ -163,7 +257,10 @@ def config(scope=None): config_cmd_line.append(scope) config_cmd_line.append('--format=json') try: - return json.loads(subprocess.check_output(config_cmd_line)) + config_data = json.loads(subprocess.check_output(config_cmd_line)) + if scope is not None: + return config_data + return Config(config_data) except ValueError: return None diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cfd26847..186147f6 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -12,6 +12,7 @@ import random import string import subprocess import hashlib +import apt_pkg from collections import OrderedDict @@ -295,3 +296,16 @@ def get_nic_hwaddr(nic): if 'link/ether' in words: hwaddr = words[words.index('link/ether') + 1] return hwaddr + + +def cmp_pkgrevno(package, revno, pkgcache=None): + '''Compare supplied revno with the revno of the installed package + 1 => Installed revno is greater than supplied arg + 0 => Installed revno is the same as supplied arg + -1 => Installed revno is less than supplied arg + ''' + if not pkgcache: + apt_pkg.init() + pkgcache = apt_pkg.Cache() + pkg = pkgcache[package] + return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index dce7db4c..e1e17dae 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -1,4 +1,5 @@ import importlib +import time from yaml import safe_load from charmhelpers.core.host import ( lsb_release @@ -15,6 +16,7 @@ from charmhelpers.core.hookenv import ( import apt_pkg import os + CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main """ @@ -56,10 +58,62 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/icehouse': 'precise-proposed/icehouse', } +# The order of this list is very important. Handlers should be listed in from +# least- to most-specific URL matching. +FETCH_HANDLERS = ( + 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', + 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', +) + +APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. +APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. +APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. + + +class SourceConfigError(Exception): + pass + + +class UnhandledSource(Exception): + pass + + +class AptLockError(Exception): + pass + + +class BaseFetchHandler(object): + + """Base class for FetchHandler implementations in fetch plugins""" + + def can_handle(self, source): + """Returns True if the source can be handled. Otherwise returns + a string explaining why it cannot""" + return "Wrong source type" + + def install(self, source): + """Try to download and unpack the source. Return the path to the + unpacked files or raise UnhandledSource.""" + raise UnhandledSource("Wrong source type {}".format(source)) + + def parse_url(self, url): + return urlparse(url) + + def base_url(self, url): + """Return url without querystring or fragment""" + parts = list(self.parse_url(url)) + parts[4:] = ['' for i in parts[4:]] + return urlunparse(parts) + def filter_installed_packages(packages): """Returns a list of packages that require installation""" apt_pkg.init() + + # Tell apt to build an in-memory cache to prevent race conditions (if + # another process is already building the cache). + apt_pkg.config.set("Dir::Cache::pkgcache", "") + cache = apt_pkg.Cache() _pkgs = [] for package in packages: @@ -87,14 +141,7 @@ def apt_install(packages, options=None, fatal=False): cmd.extend(packages) log("Installing {} with options: {}".format(packages, options)) - env = os.environ.copy() - if 'DEBIAN_FRONTEND' not in env: - env['DEBIAN_FRONTEND'] = 'noninteractive' - - if fatal: - subprocess.check_call(cmd, env=env) - else: - subprocess.call(cmd, env=env) + _run_apt_command(cmd, fatal) def apt_upgrade(options=None, fatal=False, dist=False): @@ -109,24 +156,13 @@ def apt_upgrade(options=None, fatal=False, dist=False): else: cmd.append('upgrade') log("Upgrading with options: {}".format(options)) - - env = os.environ.copy() - if 'DEBIAN_FRONTEND' not in env: - env['DEBIAN_FRONTEND'] = 'noninteractive' - - if fatal: - subprocess.check_call(cmd, env=env) - else: - subprocess.call(cmd, env=env) + _run_apt_command(cmd, fatal) def apt_update(fatal=False): """Update local apt cache""" cmd = ['apt-get', 'update'] - if fatal: - subprocess.check_call(cmd) - else: - subprocess.call(cmd) + _run_apt_command(cmd, fatal) def apt_purge(packages, fatal=False): @@ -137,10 +173,7 @@ def apt_purge(packages, fatal=False): else: cmd.extend(packages) log("Purging {}".format(packages)) - if fatal: - subprocess.check_call(cmd) - else: - subprocess.call(cmd) + _run_apt_command(cmd, fatal) def apt_hold(packages, fatal=False): @@ -151,6 +184,7 @@ def apt_hold(packages, fatal=False): else: cmd.extend(packages) log("Holding {}".format(packages)) + if fatal: subprocess.check_call(cmd) else: @@ -188,10 +222,6 @@ def add_source(source, key=None): key]) -class SourceConfigError(Exception): - pass - - def configure_sources(update=False, sources_var='install_sources', keys_var='install_keys'): @@ -224,17 +254,6 @@ def configure_sources(update=False, if update: apt_update(fatal=True) -# The order of this list is very important. Handlers should be listed in from -# least- to most-specific URL matching. -FETCH_HANDLERS = ( - 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', - 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', -) - - -class UnhandledSource(Exception): - pass - def install_remote(source): """ @@ -265,30 +284,6 @@ def install_from_config(config_var_name): return install_remote(source) -class BaseFetchHandler(object): - - """Base class for FetchHandler implementations in fetch plugins""" - - def can_handle(self, source): - """Returns True if the source can be handled. Otherwise returns - a string explaining why it cannot""" - return "Wrong source type" - - def install(self, source): - """Try to download and unpack the source. Return the path to the - unpacked files or raise UnhandledSource.""" - raise UnhandledSource("Wrong source type {}".format(source)) - - def parse_url(self, url): - return urlparse(url) - - def base_url(self, url): - """Return url without querystring or fragment""" - parts = list(self.parse_url(url)) - parts[4:] = ['' for i in parts[4:]] - return urlunparse(parts) - - def plugins(fetch_handlers=None): if not fetch_handlers: fetch_handlers = FETCH_HANDLERS @@ -306,3 +301,40 @@ def plugins(fetch_handlers=None): log("FetchHandler {} not found, skipping plugin".format( handler_name)) return plugin_list + + +def _run_apt_command(cmd, fatal=False): + """ + Run an APT command, checking output and retrying if the fatal flag is set + to True. + + :param: cmd: str: The apt command to run. + :param: fatal: bool: Whether the command's output should be checked and + retried. + """ + env = os.environ.copy() + + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + + if fatal: + retry_count = 0 + result = None + + # If the command is considered "fatal", we need to retry if the apt + # lock was not acquired. + + while result is None or result == APT_NO_LOCK: + try: + result = subprocess.check_call(cmd, env=env) + except subprocess.CalledProcessError, e: + retry_count = retry_count + 1 + if retry_count > APT_NO_LOCK_RETRY_COUNT: + raise + result = e.returncode + log("Couldn't acquire DPKG lock. Will retry in {} seconds." + "".format(APT_NO_LOCK_RETRY_DELAY)) + time.sleep(APT_NO_LOCK_RETRY_DELAY) + + else: + subprocess.call(cmd, env=env) From 11c194da99b5e35978ec1689425c593cc3d99b5c Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 21 May 2014 11:13:58 +0100 Subject: [PATCH 2/3] [trivial] Ensure lint and test pass before publish --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c65c88b4..75a00e8c 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,6 @@ test: sync: @charm-helper-sync -c charm-helpers.yaml -publish: +publish: lint test bzr push lp:charms/keystone bzr push lp:charms/trusty/keystone From 149b76c51ad9f0a005847dca22be210a0d02d6cb Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 5 Jun 2014 16:02:11 +0100 Subject: [PATCH 3/3] [mikemc,r=james-page] Add product-streams service type for stream data. --- hooks/keystone_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 338f9bbd..03ab8ac0 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -168,6 +168,10 @@ valid_services = { "heat-cfn": { "type": "cloudformation", "desc": "Heat CloudFormation API" + }, + "image-stream": { + "type": "product-streams", + "desc": "Ubuntu Product Streams" } }