openstack/charm-swift-storage
Code Issues Proposed changes

[tribaal,r=james-page,t=james-page]

Browse Source 
Resync helpers to pickup fixes for apt lock races and better block device detection and handling.
This commit is contained in:
James Page 2014-05-21 11:08:22 +01:00
commit b6e58cd003
9 changed files with 255 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -13,6 +13,6 @@ test:
sync:
@charm-helper-sync -c charm-helpers.yaml
publish:
publish: lint test
bzr push lp:charms/swift-storage
bzr push lp:charms/trusty/swift-storage

2
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -570,7 +570,7 @@ class NeutronContext(OSContextGenerator):
if self.plugin == 'ovs':
ctxt.update(self.ovs_ctxt())
elif self.plugin == 'nvp':
elif self.plugin in ['nvp', 'nsx']:
ctxt.update(self.nvp_ctxt())
alchemy_flags = config('neutron-alchemy-flags')

18
hooks/charmhelpers/contrib/openstack/neutron.py
View File

@ -114,14 +114,30 @@ def neutron_plugins():
'server_packages': ['neutron-server',
'neutron-plugin-nicira'],
'server_services': ['neutron-server']
},
'nsx': {
'config': '/etc/neutron/plugins/vmware/nsx.ini',
'driver': 'vmware',
'contexts': [
context.SharedDBContext(user=config('neutron-database-user'),
database=config('neutron-database'),
relation_prefix='neutron',
ssl_dir=NEUTRON_CONF_DIR)],
'services': [],
'packages': [],
'server_packages': ['neutron-server',
'neutron-plugin-vmware'],
'server_services': ['neutron-server']
}
}
# NOTE: patch in ml2 plugin for icehouse onwards
if release >= 'icehouse':
# NOTE: patch in ml2 plugin for icehouse onwards
plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
plugins['ovs']['server_packages'] = ['neutron-server',
'neutron-plugin-ml2']
# NOTE: patch in vmware renames nvp->nsx for icehouse onwards
plugins['nvp'] = plugins['nsx']
return plugins

9
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -131,6 +131,11 @@ def get_os_version_codename(codename):
def get_os_codename_package(package, fatal=True):
'''Derive OpenStack release codename from an installed package.'''
apt.init()
# Tell apt to build an in-memory cache to prevent race conditions (if
# another process is already building the cache).
apt.config.set("Dir::Cache::pkgcache", "")
cache = apt.Cache()
try:
@ -183,7 +188,7 @@ def get_os_version_package(pkg, fatal=True):
if cname == codename:
return version
#e = "Could not determine OpenStack version for package: %s" % pkg
#error_out(e)
# error_out(e)
os_rel = None
@ -401,6 +406,8 @@ def ns_query(address):
rtype = 'PTR'
elif isinstance(address, basestring):
rtype = 'A'
else:
return None
answers = dns.resolver.query(address, rtype)
if answers:

2
hooks/charmhelpers/contrib/storage/linux/lvm.py
View File

@ -62,7 +62,7 @@ def list_lvm_volume_group(block_device):
pvd = check_output(['pvdisplay', block_device]).splitlines()
for l in pvd:
if l.strip().startswith('VG Name'):
vg = ' '.join(l.split()).split(' ').pop()
vg = ' '.join(l.strip().split()[2:])
return vg

24
hooks/charmhelpers/contrib/storage/linux/utils.py
View File

@ -1,4 +1,5 @@
from os import stat
import os
import re
from stat import S_ISBLK
from subprocess import (
@ -14,7 +15,9 @@ def is_block_device(path):
:returns: boolean: True if path is a block device, False if not.
'''
return S_ISBLK(stat(path).st_mode)
if not os.path.exists(path):
return False
return S_ISBLK(os.stat(path).st_mode)
def zap_disk(block_device):
@ -29,7 +32,18 @@ def zap_disk(block_device):
'--clear', block_device])
dev_end = check_output(['blockdev', '--getsz', block_device])
gpt_end = int(dev_end.split()[0]) - 100
check_call(['dd', 'if=/dev/zero', 'of=%s'%(block_device),
check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
'bs=1M', 'count=1'])
check_call(['dd', 'if=/dev/zero', 'of=%s'%(block_device),
'bs=512', 'count=100', 'seek=%s'%(gpt_end)])
check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
'bs=512', 'count=100', 'seek=%s' % (gpt_end)])
def is_device_mounted(device):
'''Given a device path, return True if that device is mounted, and False
if it isn't.
:param device: str: Full path of the device to check.
:returns: boolean: True if the path represents a mounted device, False if
it doesn't.
'''
out = check_output(['mount'])
return bool(re.search(device + r"[0-9]+\b", out))

99
hooks/charmhelpers/core/hookenv.py
View File

@ -155,6 +155,100 @@ def hook_name():
return os.path.basename(sys.argv[0])
class Config(dict):
"""A Juju charm config dictionary that can write itself to
disk (as json) and track which values have changed since
the previous hook invocation.
Do not instantiate this object directly - instead call
``hookenv.config()``
Example usage::
>>> # inside a hook
>>> from charmhelpers.core import hookenv
>>> config = hookenv.config()
>>> config['foo']
'bar'
>>> config['mykey'] = 'myval'
>>> config.save()
>>> # user runs `juju set mycharm foo=baz`
>>> # now we're inside subsequent config-changed hook
>>> config = hookenv.config()
>>> config['foo']
'baz'
>>> # test to see if this val has changed since last hook
>>> config.changed('foo')
True
>>> # what was the previous value?
>>> config.previous('foo')
'bar'
>>> # keys/values that we add are preserved across hooks
>>> config['mykey']
'myval'
>>> # don't forget to save at the end of hook!
>>> config.save()
"""
CONFIG_FILE_NAME = '.juju-persistent-config'
def __init__(self, *args, **kw):
super(Config, self).__init__(*args, **kw)
self._prev_dict = None
self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
if os.path.exists(self.path):
self.load_previous()
def load_previous(self, path=None):
"""Load previous copy of config from disk so that current values
can be compared to previous values.
:param path:
File path from which to load the previous config. If `None`,
config is loaded from the default location. If `path` is
specified, subsequent `save()` calls will write to the same
path.
"""
self.path = path or self.path
with open(self.path) as f:
self._prev_dict = json.load(f)
def changed(self, key):
"""Return true if the value for this key has changed since
the last save.
"""
if self._prev_dict is None:
return True
return self.previous(key) != self.get(key)
def previous(self, key):
"""Return previous value for this key, or None if there
is no "previous" value.
"""
if self._prev_dict:
return self._prev_dict.get(key)
return None
def save(self):
"""Save this config to disk.
Preserves items in _prev_dict that do not exist in self.
"""
if self._prev_dict:
for k, v in self._prev_dict.iteritems():
if k not in self:
self[k] = v
with open(self.path, 'w') as f:
json.dump(self, f)
@cached
def config(scope=None):
"""Juju charm configuration"""
@ -163,7 +257,10 @@ def config(scope=None):
config_cmd_line.append(scope)
config_cmd_line.append('--format=json')
try:
return json.loads(subprocess.check_output(config_cmd_line))
config_data = json.loads(subprocess.check_output(config_cmd_line))
if scope is not None:
return config_data
return Config(config_data)
except ValueError:
return None

14
hooks/charmhelpers/core/host.py
View File

@ -12,6 +12,7 @@ import random
import string
import subprocess
import hashlib
import apt_pkg
from collections import OrderedDict
@ -295,3 +296,16 @@ def get_nic_hwaddr(nic):
if 'link/ether' in words:
hwaddr = words[words.index('link/ether') + 1]
return hwaddr
def cmp_pkgrevno(package, revno, pkgcache=None):
'''Compare supplied revno with the revno of the installed package
1 => Installed revno is greater than supplied arg
0 => Installed revno is the same as supplied arg
-1 => Installed revno is less than supplied arg
'''
if not pkgcache:
apt_pkg.init()
pkgcache = apt_pkg.Cache()
pkg = pkgcache[package]
return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

160
hooks/charmhelpers/fetch/__init__.py
View File

@ -1,4 +1,5 @@
import importlib
import time
from yaml import safe_load
from charmhelpers.core.host import (
lsb_release
@ -15,6 +16,7 @@ from charmhelpers.core.hookenv import (
import apt_pkg
import os
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
"""
@ -56,10 +58,62 @@ CLOUD_ARCHIVE_POCKETS = {
'precise-proposed/icehouse': 'precise-proposed/icehouse',
}
# The order of this list is very important. Handlers should be listed in from
# least- to most-specific URL matching.
FETCH_HANDLERS = (
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
)
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
class SourceConfigError(Exception):
pass
class UnhandledSource(Exception):
pass
class AptLockError(Exception):
pass
class BaseFetchHandler(object):
"""Base class for FetchHandler implementations in fetch plugins"""
def can_handle(self, source):
"""Returns True if the source can be handled. Otherwise returns
a string explaining why it cannot"""
return "Wrong source type"
def install(self, source):
"""Try to download and unpack the source. Return the path to the
unpacked files or raise UnhandledSource."""
raise UnhandledSource("Wrong source type {}".format(source))
def parse_url(self, url):
return urlparse(url)
def base_url(self, url):
"""Return url without querystring or fragment"""
parts = list(self.parse_url(url))
parts[4:] = ['' for i in parts[4:]]
return urlunparse(parts)
def filter_installed_packages(packages):
"""Returns a list of packages that require installation"""
apt_pkg.init()
# Tell apt to build an in-memory cache to prevent race conditions (if
# another process is already building the cache).
apt_pkg.config.set("Dir::Cache::pkgcache", "")
cache = apt_pkg.Cache()
_pkgs = []
for package in packages:
@ -87,14 +141,7 @@ def apt_install(packages, options=None, fatal=False):
cmd.extend(packages)
log("Installing {} with options: {}".format(packages,
options))
env = os.environ.copy()
if 'DEBIAN_FRONTEND' not in env:
env['DEBIAN_FRONTEND'] = 'noninteractive'
if fatal:
subprocess.check_call(cmd, env=env)
else:
subprocess.call(cmd, env=env)
_run_apt_command(cmd, fatal)
def apt_upgrade(options=None, fatal=False, dist=False):
@ -109,24 +156,13 @@ def apt_upgrade(options=None, fatal=False, dist=False):
else:
cmd.append('upgrade')
log("Upgrading with options: {}".format(options))
env = os.environ.copy()
if 'DEBIAN_FRONTEND' not in env:
env['DEBIAN_FRONTEND'] = 'noninteractive'
if fatal:
subprocess.check_call(cmd, env=env)
else:
subprocess.call(cmd, env=env)
_run_apt_command(cmd, fatal)
def apt_update(fatal=False):
"""Update local apt cache"""
cmd = ['apt-get', 'update']
if fatal:
subprocess.check_call(cmd)
else:
subprocess.call(cmd)
_run_apt_command(cmd, fatal)
def apt_purge(packages, fatal=False):
@ -137,10 +173,7 @@ def apt_purge(packages, fatal=False):
else:
cmd.extend(packages)
log("Purging {}".format(packages))
if fatal:
subprocess.check_call(cmd)
else:
subprocess.call(cmd)
_run_apt_command(cmd, fatal)
def apt_hold(packages, fatal=False):
@ -151,6 +184,7 @@ def apt_hold(packages, fatal=False):
else:
cmd.extend(packages)
log("Holding {}".format(packages))
if fatal:
subprocess.check_call(cmd)
else:
@ -188,10 +222,6 @@ def add_source(source, key=None):
key])
class SourceConfigError(Exception):
pass
def configure_sources(update=False,
sources_var='install_sources',
keys_var='install_keys'):
@ -224,17 +254,6 @@ def configure_sources(update=False,
if update:
apt_update(fatal=True)
# The order of this list is very important. Handlers should be listed in from
# least- to most-specific URL matching.
FETCH_HANDLERS = (
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
)
class UnhandledSource(Exception):
pass
def install_remote(source):
"""
@ -265,30 +284,6 @@ def install_from_config(config_var_name):
return install_remote(source)
class BaseFetchHandler(object):
"""Base class for FetchHandler implementations in fetch plugins"""
def can_handle(self, source):
"""Returns True if the source can be handled. Otherwise returns
a string explaining why it cannot"""
return "Wrong source type"
def install(self, source):
"""Try to download and unpack the source. Return the path to the
unpacked files or raise UnhandledSource."""
raise UnhandledSource("Wrong source type {}".format(source))
def parse_url(self, url):
return urlparse(url)
def base_url(self, url):
"""Return url without querystring or fragment"""
parts = list(self.parse_url(url))
parts[4:] = ['' for i in parts[4:]]
return urlunparse(parts)
def plugins(fetch_handlers=None):
if not fetch_handlers:
fetch_handlers = FETCH_HANDLERS
@ -306,3 +301,40 @@ def plugins(fetch_handlers=None):
log("FetchHandler {} not found, skipping plugin".format(
handler_name))
return plugin_list
def _run_apt_command(cmd, fatal=False):
"""
Run an APT command, checking output and retrying if the fatal flag is set
to True.
:param: cmd: str: The apt command to run.
:param: fatal: bool: Whether the command's output should be checked and
retried.
"""
env = os.environ.copy()
if 'DEBIAN_FRONTEND' not in env:
env['DEBIAN_FRONTEND'] = 'noninteractive'
if fatal:
retry_count = 0
result = None
# If the command is considered "fatal", we need to retry if the apt
# lock was not acquired.
while result is None or result == APT_NO_LOCK:
try:
result = subprocess.check_call(cmd, env=env)
except subprocess.CalledProcessError, e:
retry_count = retry_count + 1
if retry_count > APT_NO_LOCK_RETRY_COUNT:
raise
result = e.returncode
log("Couldn't acquire DPKG lock. Will retry in {} seconds."
"".format(APT_NO_LOCK_RETRY_DELAY))
time.sleep(APT_NO_LOCK_RETRY_DELAY)
else:
subprocess.call(cmd, env=env)