Setup Ceph default RBD features
Nova-lxd requires that our Ceph images only contain the features supported by the kernel RBD driver, and a discussion on the dev mailing list suggests that 1 should work fine as the driver level This commit contains a charmhelpers sync to bring in the new flag to support configuration sent from the ceph charms. Change-Id: Ifb20000fb8ec7c54e752fdb3acdc16043637a81d
This commit is contained in:
Chris MacNaughton
parent
ce7f164e7f
commit
3c90647581
@ -125,7 +125,7 @@ class CheckException(Exception):
|
||||
|
||||
|
||||
class Check(object):
|
||||
shortname_re = '[A-Za-z0-9-_]+$'
|
||||
shortname_re = '[A-Za-z0-9-_.]+$'
|
||||
service_template = ("""
|
||||
#---------------------------------------------------
|
||||
# This file is Juju managed
|
||||
|
||||
@ -243,11 +243,11 @@ def is_ipv6_disabled():
|
||||
try:
|
||||
result = subprocess.check_output(
|
||||
['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
|
||||
stderr=subprocess.STDOUT)
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True)
|
||||
except subprocess.CalledProcessError:
|
||||
return True
|
||||
if six.PY3:
|
||||
result = result.decode('UTF-8')
|
||||
|
||||
return "net.ipv6.conf.all.disable_ipv6 = 1" in result
|
||||
|
||||
|
||||
|
||||
@ -97,6 +97,7 @@ from charmhelpers.contrib.openstack.utils import (
|
||||
git_determine_usr_bin,
|
||||
git_determine_python_path,
|
||||
enable_memcache,
|
||||
snap_install_requested,
|
||||
)
|
||||
from charmhelpers.core.unitdata import kv
|
||||
|
||||
@ -244,6 +245,11 @@ class SharedDBContext(OSContextGenerator):
|
||||
'database_password': rdata.get(password_setting),
|
||||
'database_type': 'mysql'
|
||||
}
|
||||
# Note(coreycb): We can drop mysql+pymysql if we want when the
|
||||
# following review lands, though it seems mysql+pymysql would
|
||||
# be preferred. https://review.openstack.org/#/c/462190/
|
||||
if snap_install_requested():
|
||||
ctxt['database_type'] = 'mysql+pymysql'
|
||||
if self.context_complete(ctxt):
|
||||
db_ssl(rdata, ctxt, self.ssl_dir)
|
||||
return ctxt
|
||||
@ -510,6 +516,10 @@ class CephContext(OSContextGenerator):
|
||||
ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
|
||||
if not ctxt.get('key'):
|
||||
ctxt['key'] = relation_get('key', rid=rid, unit=unit)
|
||||
if not ctxt.get('rbd_features'):
|
||||
default_features = relation_get('rbd-features', rid=rid, unit=unit)
|
||||
if default_features is not None:
|
||||
ctxt['rbd_features'] = default_features
|
||||
|
||||
ceph_addrs = relation_get('ceph-public-address', rid=rid,
|
||||
unit=unit)
|
||||
@ -1397,6 +1407,18 @@ class NeutronAPIContext(OSContextGenerator):
|
||||
'rel_key': 'dns-domain',
|
||||
'default': None,
|
||||
},
|
||||
'polling_interval': {
|
||||
'rel_key': 'polling-interval',
|
||||
'default': 2,
|
||||
},
|
||||
'rpc_response_timeout': {
|
||||
'rel_key': 'rpc-response-timeout',
|
||||
'default': 60,
|
||||
},
|
||||
'report_interval': {
|
||||
'rel_key': 'report-interval',
|
||||
'default': 30,
|
||||
},
|
||||
}
|
||||
ctxt = self.get_neutron_options({})
|
||||
for rid in relation_ids('neutron-plugin-api'):
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
###############################################################################
|
||||
# [ WARNING ]
|
||||
# cinder configuration file maintained by Juju
|
||||
# ceph configuration file maintained by Juju
|
||||
# local changes may be overwritten.
|
||||
###############################################################################
|
||||
[global]
|
||||
@ -12,6 +12,9 @@ mon host = {{ mon_hosts }}
|
||||
log to syslog = {{ use_syslog }}
|
||||
err to syslog = {{ use_syslog }}
|
||||
clog to syslog = {{ use_syslog }}
|
||||
{% if rbd_features %}
|
||||
rbd default features = {{ rbd_features }}
|
||||
{% endif %}
|
||||
|
||||
[client]
|
||||
{% if rbd_client_cache_settings -%}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
global
|
||||
log {{ local_host }} local0
|
||||
log {{ local_host }} local1 notice
|
||||
log /var/lib/haproxy/dev/log local0
|
||||
log /var/lib/haproxy/dev/log local1 notice
|
||||
maxconn 20000
|
||||
user haproxy
|
||||
group haproxy
|
||||
|
||||
@ -0,0 +1,8 @@
|
||||
{% if transport_url -%}
|
||||
[oslo_messaging_notifications]
|
||||
driver = messagingv2
|
||||
transport_url = {{ transport_url }}
|
||||
{% if notification_topics -%}
|
||||
topics = {{ notification_topics }}
|
||||
{% endif -%}
|
||||
{% endif -%}
|
||||
@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import (
|
||||
status_set,
|
||||
hook_name,
|
||||
application_version_set,
|
||||
cached,
|
||||
)
|
||||
|
||||
from charmhelpers.core.strutils import BasicStringComparator
|
||||
@ -90,6 +91,13 @@ from charmhelpers.fetch import (
|
||||
GPGKeyError,
|
||||
get_upstream_version
|
||||
)
|
||||
|
||||
from charmhelpers.fetch.snap import (
|
||||
snap_install,
|
||||
snap_refresh,
|
||||
SNAP_CHANNELS,
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
|
||||
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
|
||||
from charmhelpers.contrib.openstack.exceptions import OSContextError
|
||||
@ -327,8 +335,10 @@ def get_os_codename_install_source(src):
|
||||
return ca_rel
|
||||
|
||||
# Best guess match based on deb string provided
|
||||
if src.startswith('deb') or src.startswith('ppa'):
|
||||
for k, v in six.iteritems(OPENSTACK_CODENAMES):
|
||||
if (src.startswith('deb') or
|
||||
src.startswith('ppa') or
|
||||
src.startswith('snap')):
|
||||
for v in OPENSTACK_CODENAMES.values():
|
||||
if v in src:
|
||||
return v
|
||||
|
||||
@ -397,6 +407,19 @@ def get_swift_codename(version):
|
||||
|
||||
def get_os_codename_package(package, fatal=True):
|
||||
'''Derive OpenStack release codename from an installed package.'''
|
||||
|
||||
if snap_install_requested():
|
||||
cmd = ['snap', 'list', package]
|
||||
try:
|
||||
out = subprocess.check_output(cmd)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return None
|
||||
lines = out.split('\n')
|
||||
for line in lines:
|
||||
if package in line:
|
||||
# Second item in list is Version
|
||||
return line.split()[1]
|
||||
|
||||
import apt_pkg as apt
|
||||
|
||||
cache = apt_cache()
|
||||
@ -613,6 +636,9 @@ def openstack_upgrade_available(package):
|
||||
import apt_pkg as apt
|
||||
src = config('openstack-origin')
|
||||
cur_vers = get_os_version_package(package)
|
||||
if not cur_vers:
|
||||
# The package has not been installed yet do not attempt upgrade
|
||||
return False
|
||||
if "swift" in package:
|
||||
codename = get_os_codename_install_source(src)
|
||||
avail_vers = get_os_version_codename_swift(codename)
|
||||
@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False,
|
||||
return wrap
|
||||
|
||||
|
||||
def ordered(orderme):
|
||||
"""Converts the provided dictionary into a collections.OrderedDict.
|
||||
|
||||
The items in the returned OrderedDict will be inserted based on the
|
||||
natural sort order of the keys. Nested dictionaries will also be sorted
|
||||
in order to ensure fully predictable ordering.
|
||||
|
||||
:param orderme: the dict to order
|
||||
:return: collections.OrderedDict
|
||||
:raises: ValueError: if `orderme` isn't a dict instance.
|
||||
"""
|
||||
if not isinstance(orderme, dict):
|
||||
raise ValueError('argument must be a dict type')
|
||||
|
||||
result = OrderedDict()
|
||||
for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]):
|
||||
if isinstance(v, dict):
|
||||
result[k] = ordered(v)
|
||||
else:
|
||||
result[k] = v
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def config_flags_parser(config_flags):
|
||||
"""Parses config flags string into dict.
|
||||
|
||||
@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags):
|
||||
example, a string in the format of 'key1=value1, key2=value2' will
|
||||
return a dict of:
|
||||
|
||||
{'key1': 'value1',
|
||||
'key2': 'value2'}.
|
||||
{'key1': 'value1', 'key2': 'value2'}.
|
||||
|
||||
2. A string in the above format, but supporting a comma-delimited list
|
||||
of values for the same key. For example, a string in the format of
|
||||
'key1=value1, key2=value3,value4,value5' will return a dict of:
|
||||
|
||||
{'key1', 'value1',
|
||||
'key2', 'value2,value3,value4'}
|
||||
{'key1': 'value1', 'key2': 'value2,value3,value4'}
|
||||
|
||||
3. A string containing a colon character (:) prior to an equal
|
||||
character (=) will be treated as yaml and parsed as such. This can be
|
||||
@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags):
|
||||
equals = config_flags.find('=')
|
||||
if colon > 0:
|
||||
if colon < equals or equals < 0:
|
||||
return yaml.safe_load(config_flags)
|
||||
return ordered(yaml.safe_load(config_flags))
|
||||
|
||||
if config_flags.find('==') >= 0:
|
||||
juju_log("config_flags is not in expected format (key=value)",
|
||||
@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags):
|
||||
# split on '='.
|
||||
split = config_flags.strip(' =').split('=')
|
||||
limit = len(split)
|
||||
flags = {}
|
||||
flags = OrderedDict()
|
||||
for i in range(0, limit - 1):
|
||||
current = split[i]
|
||||
next = split[i + 1]
|
||||
@ -1982,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None):
|
||||
if enable_memcache(source=source, release=release):
|
||||
packages.extend(['memcached', 'python-memcache'])
|
||||
return packages
|
||||
|
||||
|
||||
def update_json_file(filename, items):
|
||||
"""Updates the json `filename` with a given dict.
|
||||
:param filename: json filename (i.e.: /etc/glance/policy.json)
|
||||
:param items: dict of items to update
|
||||
"""
|
||||
with open(filename) as fd:
|
||||
policy = json.load(fd)
|
||||
policy.update(items)
|
||||
with open(filename, "w") as fd:
|
||||
fd.write(json.dumps(policy, indent=4))
|
||||
|
||||
|
||||
@cached
|
||||
def snap_install_requested():
|
||||
""" Determine if installing from snaps
|
||||
|
||||
If openstack-origin is of the form snap:channel-series-release
|
||||
and channel is in SNAPS_CHANNELS return True.
|
||||
"""
|
||||
origin = config('openstack-origin') or ""
|
||||
if not origin.startswith('snap:'):
|
||||
return False
|
||||
|
||||
_src = origin[5:]
|
||||
channel, series, release = _src.split('-')
|
||||
if channel.lower() in SNAP_CHANNELS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
|
||||
"""Generate a dictionary of snap install information from origin
|
||||
|
||||
@param snaps: List of snaps
|
||||
@param src: String of openstack-origin or source of the form
|
||||
snap:channel-series-track
|
||||
@param mode: String classic, devmode or jailmode
|
||||
@returns: Dictionary of snaps with channels and modes
|
||||
"""
|
||||
|
||||
if not src.startswith('snap:'):
|
||||
juju_log("Snap source is not a snap origin", 'WARN')
|
||||
return {}
|
||||
|
||||
_src = src[5:]
|
||||
_channel, _series, _release = _src.split('-')
|
||||
channel = '--channel={}/{}'.format(_release, _channel)
|
||||
|
||||
return {snap: {'channel': channel, 'mode': mode}
|
||||
for snap in snaps}
|
||||
|
||||
|
||||
def install_os_snaps(snaps, refresh=False):
|
||||
"""Install OpenStack snaps from channel and with mode
|
||||
|
||||
@param snaps: Dictionary of snaps with channels and modes of the form:
|
||||
{'snap_name': {'channel': 'snap_channel',
|
||||
'mode': 'snap_mode'}}
|
||||
Where channel a snapstore channel and mode is --classic, --devmode or
|
||||
--jailmode.
|
||||
@param post_snap_install: Callback function to run after snaps have been
|
||||
installed
|
||||
"""
|
||||
|
||||
def _ensure_flag(flag):
|
||||
if flag.startswith('--'):
|
||||
return flag
|
||||
return '--{}'.format(flag)
|
||||
|
||||
if refresh:
|
||||
for snap in snaps.keys():
|
||||
snap_refresh(snap,
|
||||
_ensure_flag(snaps[snap]['channel']),
|
||||
_ensure_flag(snaps[snap]['mode']))
|
||||
else:
|
||||
for snap in snaps.keys():
|
||||
snap_install(snap,
|
||||
_ensure_flag(snaps[snap]['channel']),
|
||||
_ensure_flag(snaps[snap]['mode']))
|
||||
|
||||
74
hooks/charmhelpers/contrib/storage/linux/bcache.py
Normal file
74
hooks/charmhelpers/contrib/storage/linux/bcache.py
Normal file
@ -0,0 +1,74 @@
|
||||
# Copyright 2017 Canonical Limited.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
import json
|
||||
|
||||
from charmhelpers.core.hookenv import log
|
||||
|
||||
stats_intervals = ['stats_day', 'stats_five_minute',
|
||||
'stats_hour', 'stats_total']
|
||||
|
||||
SYSFS = '/sys'
|
||||
|
||||
|
||||
class Bcache(object):
|
||||
"""Bcache behaviour
|
||||
"""
|
||||
|
||||
def __init__(self, cachepath):
|
||||
self.cachepath = cachepath
|
||||
|
||||
@classmethod
|
||||
def fromdevice(cls, devname):
|
||||
return cls('{}/block/{}/bcache'.format(SYSFS, devname))
|
||||
|
||||
def __str__(self):
|
||||
return self.cachepath
|
||||
|
||||
def get_stats(self, interval):
|
||||
"""Get cache stats
|
||||
"""
|
||||
intervaldir = 'stats_{}'.format(interval)
|
||||
path = "{}/{}".format(self.cachepath, intervaldir)
|
||||
out = dict()
|
||||
for elem in os.listdir(path):
|
||||
out[elem] = open('{}/{}'.format(path, elem)).read().strip()
|
||||
return out
|
||||
|
||||
|
||||
def get_bcache_fs():
|
||||
"""Return all cache sets
|
||||
"""
|
||||
cachesetroot = "{}/fs/bcache".format(SYSFS)
|
||||
try:
|
||||
dirs = os.listdir(cachesetroot)
|
||||
except OSError:
|
||||
log("No bcache fs found")
|
||||
return []
|
||||
cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')])
|
||||
return cacheset
|
||||
|
||||
|
||||
def get_stats_action(cachespec, interval):
|
||||
"""Action for getting bcache statistics for a given cachespec.
|
||||
Cachespec can either be a device name, eg. 'sdb', which will retrieve
|
||||
cache stats for the given device, or 'global', which will retrieve stats
|
||||
for all cachesets
|
||||
"""
|
||||
if cachespec == 'global':
|
||||
caches = get_bcache_fs()
|
||||
else:
|
||||
caches = [Bcache.fromdevice(cachespec)]
|
||||
res = dict((c.cachepath, c.get_stats(interval)) for c in caches)
|
||||
return json.dumps(res, indent=4, separators=(',', ': '))
|
||||
@ -1372,7 +1372,7 @@ class CephConfContext(object):
|
||||
return {}
|
||||
|
||||
conf = config_flags_parser(conf)
|
||||
if type(conf) != dict:
|
||||
if not isinstance(conf, dict):
|
||||
log("Provided config-flags is not a dictionary - ignoring",
|
||||
level=WARNING)
|
||||
return {}
|
||||
|
||||
@ -202,6 +202,27 @@ def service_name():
|
||||
return local_unit().split('/')[0]
|
||||
|
||||
|
||||
def principal_unit():
|
||||
"""Returns the principal unit of this unit, otherwise None"""
|
||||
# Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT
|
||||
principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None)
|
||||
# If it's empty, then this unit is the principal
|
||||
if principal_unit == '':
|
||||
return os.environ['JUJU_UNIT_NAME']
|
||||
elif principal_unit is not None:
|
||||
return principal_unit
|
||||
# For Juju 2.1 and below, let's try work out the principle unit by
|
||||
# the various charms' metadata.yaml.
|
||||
for reltype in relation_types():
|
||||
for rid in relation_ids(reltype):
|
||||
for unit in related_units(rid):
|
||||
md = _metadata_unit(unit)
|
||||
subordinate = md.pop('subordinate', None)
|
||||
if not subordinate:
|
||||
return unit
|
||||
return None
|
||||
|
||||
|
||||
@cached
|
||||
def remote_service_name(relid=None):
|
||||
"""The remote service name for a given relation-id (or the current relation)"""
|
||||
@ -478,6 +499,21 @@ def metadata():
|
||||
return yaml.safe_load(md)
|
||||
|
||||
|
||||
def _metadata_unit(unit):
|
||||
"""Given the name of a unit (e.g. apache2/0), get the unit charm's
|
||||
metadata.yaml. Very similar to metadata() but allows us to inspect
|
||||
other units. Unit needs to be co-located, such as a subordinate or
|
||||
principal/primary.
|
||||
|
||||
:returns: metadata.yaml as a python object.
|
||||
|
||||
"""
|
||||
basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
|
||||
unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
|
||||
with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
|
||||
return yaml.safe_load(md)
|
||||
|
||||
|
||||
@cached
|
||||
def relation_types():
|
||||
"""Get a list of relation types supported by this charm"""
|
||||
|
||||
@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer:
|
||||
https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
|
||||
"""
|
||||
import subprocess
|
||||
from os import environ
|
||||
import os
|
||||
from time import sleep
|
||||
from charmhelpers.core.hookenv import log
|
||||
|
||||
__author__ = 'Joseph Borg <joseph.borg@canonical.com>'
|
||||
|
||||
SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
|
||||
# The return code for "couldn't acquire lock" in Snap
|
||||
# (hopefully this will be improved).
|
||||
SNAP_NO_LOCK = 1
|
||||
SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks.
|
||||
SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
|
||||
SNAP_CHANNELS = [
|
||||
'edge',
|
||||
'beta',
|
||||
'candidate',
|
||||
'stable',
|
||||
]
|
||||
|
||||
|
||||
class CouldNotAcquireLockException(Exception):
|
||||
@ -47,13 +55,17 @@ def _snap_exec(commands):
|
||||
|
||||
while return_code is None or return_code == SNAP_NO_LOCK:
|
||||
try:
|
||||
return_code = subprocess.check_call(['snap'] + commands, env=environ)
|
||||
return_code = subprocess.check_call(['snap'] + commands,
|
||||
env=os.environ)
|
||||
except subprocess.CalledProcessError as e:
|
||||
retry_count += + 1
|
||||
if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
|
||||
raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
|
||||
raise CouldNotAcquireLockException(
|
||||
'Could not aquire lock after {} attempts'
|
||||
.format(SNAP_NO_LOCK_RETRY_COUNT))
|
||||
return_code = e.returncode
|
||||
log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
|
||||
log('Snap failed to acquire lock, trying again in {} seconds.'
|
||||
.format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN'))
|
||||
sleep(SNAP_NO_LOCK_RETRY_DELAY)
|
||||
|
||||
return return_code
|
||||
|
||||
@ -139,7 +139,7 @@ CLOUD_ARCHIVE_POCKETS = {
|
||||
'xenial-updates/ocata': 'xenial-updates/ocata',
|
||||
'ocata/proposed': 'xenial-proposed/ocata',
|
||||
'xenial-ocata/proposed': 'xenial-proposed/ocata',
|
||||
'xenial-ocata/newton': 'xenial-proposed/ocata',
|
||||
'xenial-proposed/ocata': 'xenial-proposed/ocata',
|
||||
# Pike
|
||||
'pike': 'xenial-updates/pike',
|
||||
'xenial-pike': 'xenial-updates/pike',
|
||||
@ -147,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = {
|
||||
'xenial-updates/pike': 'xenial-updates/pike',
|
||||
'pike/proposed': 'xenial-proposed/pike',
|
||||
'xenial-pike/proposed': 'xenial-proposed/pike',
|
||||
'xenial-pike/newton': 'xenial-proposed/pike',
|
||||
'xenial-proposed/pike': 'xenial-proposed/pike',
|
||||
# Queens
|
||||
'queens': 'xenial-updates/queens',
|
||||
'xenial-queens': 'xenial-updates/queens',
|
||||
@ -155,13 +155,13 @@ CLOUD_ARCHIVE_POCKETS = {
|
||||
'xenial-updates/queens': 'xenial-updates/queens',
|
||||
'queens/proposed': 'xenial-proposed/queens',
|
||||
'xenial-queens/proposed': 'xenial-proposed/queens',
|
||||
'xenial-queens/newton': 'xenial-proposed/queens',
|
||||
'xenial-proposed/queens': 'xenial-proposed/queens',
|
||||
}
|
||||
|
||||
|
||||
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
|
||||
CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries.
|
||||
CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times.
|
||||
CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times.
|
||||
|
||||
|
||||
def filter_installed_packages(packages):
|
||||
@ -364,6 +364,7 @@ def add_source(source, key=None, fail_invalid=False):
|
||||
(r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
|
||||
(r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
|
||||
(r"^cloud:(.*)$", _add_cloud_pocket),
|
||||
(r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check),
|
||||
])
|
||||
if source is None:
|
||||
source = ''
|
||||
|
||||
@ -243,11 +243,11 @@ def is_ipv6_disabled():
|
||||
try:
|
||||
result = subprocess.check_output(
|
||||
['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
|
||||
stderr=subprocess.STDOUT)
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True)
|
||||
except subprocess.CalledProcessError:
|
||||
return True
|
||||
if six.PY3:
|
||||
result = result.decode('UTF-8')
|
||||
|
||||
return "net.ipv6.conf.all.disable_ipv6 = 1" in result
|
||||
|
||||
|
||||
|
||||
@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import (
|
||||
status_set,
|
||||
hook_name,
|
||||
application_version_set,
|
||||
cached,
|
||||
)
|
||||
|
||||
from charmhelpers.core.strutils import BasicStringComparator
|
||||
@ -90,6 +91,13 @@ from charmhelpers.fetch import (
|
||||
GPGKeyError,
|
||||
get_upstream_version
|
||||
)
|
||||
|
||||
from charmhelpers.fetch.snap import (
|
||||
snap_install,
|
||||
snap_refresh,
|
||||
SNAP_CHANNELS,
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
|
||||
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
|
||||
from charmhelpers.contrib.openstack.exceptions import OSContextError
|
||||
@ -327,8 +335,10 @@ def get_os_codename_install_source(src):
|
||||
return ca_rel
|
||||
|
||||
# Best guess match based on deb string provided
|
||||
if src.startswith('deb') or src.startswith('ppa'):
|
||||
for k, v in six.iteritems(OPENSTACK_CODENAMES):
|
||||
if (src.startswith('deb') or
|
||||
src.startswith('ppa') or
|
||||
src.startswith('snap')):
|
||||
for v in OPENSTACK_CODENAMES.values():
|
||||
if v in src:
|
||||
return v
|
||||
|
||||
@ -397,6 +407,19 @@ def get_swift_codename(version):
|
||||
|
||||
def get_os_codename_package(package, fatal=True):
|
||||
'''Derive OpenStack release codename from an installed package.'''
|
||||
|
||||
if snap_install_requested():
|
||||
cmd = ['snap', 'list', package]
|
||||
try:
|
||||
out = subprocess.check_output(cmd)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return None
|
||||
lines = out.split('\n')
|
||||
for line in lines:
|
||||
if package in line:
|
||||
# Second item in list is Version
|
||||
return line.split()[1]
|
||||
|
||||
import apt_pkg as apt
|
||||
|
||||
cache = apt_cache()
|
||||
@ -613,6 +636,9 @@ def openstack_upgrade_available(package):
|
||||
import apt_pkg as apt
|
||||
src = config('openstack-origin')
|
||||
cur_vers = get_os_version_package(package)
|
||||
if not cur_vers:
|
||||
# The package has not been installed yet do not attempt upgrade
|
||||
return False
|
||||
if "swift" in package:
|
||||
codename = get_os_codename_install_source(src)
|
||||
avail_vers = get_os_version_codename_swift(codename)
|
||||
@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False,
|
||||
return wrap
|
||||
|
||||
|
||||
def ordered(orderme):
|
||||
"""Converts the provided dictionary into a collections.OrderedDict.
|
||||
|
||||
The items in the returned OrderedDict will be inserted based on the
|
||||
natural sort order of the keys. Nested dictionaries will also be sorted
|
||||
in order to ensure fully predictable ordering.
|
||||
|
||||
:param orderme: the dict to order
|
||||
:return: collections.OrderedDict
|
||||
:raises: ValueError: if `orderme` isn't a dict instance.
|
||||
"""
|
||||
if not isinstance(orderme, dict):
|
||||
raise ValueError('argument must be a dict type')
|
||||
|
||||
result = OrderedDict()
|
||||
for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]):
|
||||
if isinstance(v, dict):
|
||||
result[k] = ordered(v)
|
||||
else:
|
||||
result[k] = v
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def config_flags_parser(config_flags):
|
||||
"""Parses config flags string into dict.
|
||||
|
||||
@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags):
|
||||
example, a string in the format of 'key1=value1, key2=value2' will
|
||||
return a dict of:
|
||||
|
||||
{'key1': 'value1',
|
||||
'key2': 'value2'}.
|
||||
{'key1': 'value1', 'key2': 'value2'}.
|
||||
|
||||
2. A string in the above format, but supporting a comma-delimited list
|
||||
of values for the same key. For example, a string in the format of
|
||||
'key1=value1, key2=value3,value4,value5' will return a dict of:
|
||||
|
||||
{'key1', 'value1',
|
||||
'key2', 'value2,value3,value4'}
|
||||
{'key1': 'value1', 'key2': 'value2,value3,value4'}
|
||||
|
||||
3. A string containing a colon character (:) prior to an equal
|
||||
character (=) will be treated as yaml and parsed as such. This can be
|
||||
@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags):
|
||||
equals = config_flags.find('=')
|
||||
if colon > 0:
|
||||
if colon < equals or equals < 0:
|
||||
return yaml.safe_load(config_flags)
|
||||
return ordered(yaml.safe_load(config_flags))
|
||||
|
||||
if config_flags.find('==') >= 0:
|
||||
juju_log("config_flags is not in expected format (key=value)",
|
||||
@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags):
|
||||
# split on '='.
|
||||
split = config_flags.strip(' =').split('=')
|
||||
limit = len(split)
|
||||
flags = {}
|
||||
flags = OrderedDict()
|
||||
for i in range(0, limit - 1):
|
||||
current = split[i]
|
||||
next = split[i + 1]
|
||||
@ -1982,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None):
|
||||
if enable_memcache(source=source, release=release):
|
||||
packages.extend(['memcached', 'python-memcache'])
|
||||
return packages
|
||||
|
||||
|
||||
def update_json_file(filename, items):
|
||||
"""Updates the json `filename` with a given dict.
|
||||
:param filename: json filename (i.e.: /etc/glance/policy.json)
|
||||
:param items: dict of items to update
|
||||
"""
|
||||
with open(filename) as fd:
|
||||
policy = json.load(fd)
|
||||
policy.update(items)
|
||||
with open(filename, "w") as fd:
|
||||
fd.write(json.dumps(policy, indent=4))
|
||||
|
||||
|
||||
@cached
|
||||
def snap_install_requested():
|
||||
""" Determine if installing from snaps
|
||||
|
||||
If openstack-origin is of the form snap:channel-series-release
|
||||
and channel is in SNAPS_CHANNELS return True.
|
||||
"""
|
||||
origin = config('openstack-origin') or ""
|
||||
if not origin.startswith('snap:'):
|
||||
return False
|
||||
|
||||
_src = origin[5:]
|
||||
channel, series, release = _src.split('-')
|
||||
if channel.lower() in SNAP_CHANNELS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
|
||||
"""Generate a dictionary of snap install information from origin
|
||||
|
||||
@param snaps: List of snaps
|
||||
@param src: String of openstack-origin or source of the form
|
||||
snap:channel-series-track
|
||||
@param mode: String classic, devmode or jailmode
|
||||
@returns: Dictionary of snaps with channels and modes
|
||||
"""
|
||||
|
||||
if not src.startswith('snap:'):
|
||||
juju_log("Snap source is not a snap origin", 'WARN')
|
||||
return {}
|
||||
|
||||
_src = src[5:]
|
||||
_channel, _series, _release = _src.split('-')
|
||||
channel = '--channel={}/{}'.format(_release, _channel)
|
||||
|
||||
return {snap: {'channel': channel, 'mode': mode}
|
||||
for snap in snaps}
|
||||
|
||||
|
||||
def install_os_snaps(snaps, refresh=False):
|
||||
"""Install OpenStack snaps from channel and with mode
|
||||
|
||||
@param snaps: Dictionary of snaps with channels and modes of the form:
|
||||
{'snap_name': {'channel': 'snap_channel',
|
||||
'mode': 'snap_mode'}}
|
||||
Where channel a snapstore channel and mode is --classic, --devmode or
|
||||
--jailmode.
|
||||
@param post_snap_install: Callback function to run after snaps have been
|
||||
installed
|
||||
"""
|
||||
|
||||
def _ensure_flag(flag):
|
||||
if flag.startswith('--'):
|
||||
return flag
|
||||
return '--{}'.format(flag)
|
||||
|
||||
if refresh:
|
||||
for snap in snaps.keys():
|
||||
snap_refresh(snap,
|
||||
_ensure_flag(snaps[snap]['channel']),
|
||||
_ensure_flag(snaps[snap]['mode']))
|
||||
else:
|
||||
for snap in snaps.keys():
|
||||
snap_install(snap,
|
||||
_ensure_flag(snaps[snap]['channel']),
|
||||
_ensure_flag(snaps[snap]['mode']))
|
||||
|
||||
74
tests/charmhelpers/contrib/storage/linux/bcache.py
Normal file
74
tests/charmhelpers/contrib/storage/linux/bcache.py
Normal file
@ -0,0 +1,74 @@
|
||||
# Copyright 2017 Canonical Limited.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
import json
|
||||
|
||||
from charmhelpers.core.hookenv import log
|
||||
|
||||
stats_intervals = ['stats_day', 'stats_five_minute',
|
||||
'stats_hour', 'stats_total']
|
||||
|
||||
SYSFS = '/sys'
|
||||
|
||||
|
||||
class Bcache(object):
|
||||
"""Bcache behaviour
|
||||
"""
|
||||
|
||||
def __init__(self, cachepath):
|
||||
self.cachepath = cachepath
|
||||
|
||||
@classmethod
|
||||
def fromdevice(cls, devname):
|
||||
return cls('{}/block/{}/bcache'.format(SYSFS, devname))
|
||||
|
||||
def __str__(self):
|
||||
return self.cachepath
|
||||
|
||||
def get_stats(self, interval):
|
||||
"""Get cache stats
|
||||
"""
|
||||
intervaldir = 'stats_{}'.format(interval)
|
||||
path = "{}/{}".format(self.cachepath, intervaldir)
|
||||
out = dict()
|
||||
for elem in os.listdir(path):
|
||||
out[elem] = open('{}/{}'.format(path, elem)).read().strip()
|
||||
return out
|
||||
|
||||
|
||||
def get_bcache_fs():
|
||||
"""Return all cache sets
|
||||
"""
|
||||
cachesetroot = "{}/fs/bcache".format(SYSFS)
|
||||
try:
|
||||
dirs = os.listdir(cachesetroot)
|
||||
except OSError:
|
||||
log("No bcache fs found")
|
||||
return []
|
||||
cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')])
|
||||
return cacheset
|
||||
|
||||
|
||||
def get_stats_action(cachespec, interval):
|
||||
"""Action for getting bcache statistics for a given cachespec.
|
||||
Cachespec can either be a device name, eg. 'sdb', which will retrieve
|
||||
cache stats for the given device, or 'global', which will retrieve stats
|
||||
for all cachesets
|
||||
"""
|
||||
if cachespec == 'global':
|
||||
caches = get_bcache_fs()
|
||||
else:
|
||||
caches = [Bcache.fromdevice(cachespec)]
|
||||
res = dict((c.cachepath, c.get_stats(interval)) for c in caches)
|
||||
return json.dumps(res, indent=4, separators=(',', ': '))
|
||||
@ -1372,7 +1372,7 @@ class CephConfContext(object):
|
||||
return {}
|
||||
|
||||
conf = config_flags_parser(conf)
|
||||
if type(conf) != dict:
|
||||
if not isinstance(conf, dict):
|
||||
log("Provided config-flags is not a dictionary - ignoring",
|
||||
level=WARNING)
|
||||
return {}
|
||||
|
||||
@ -202,6 +202,27 @@ def service_name():
|
||||
return local_unit().split('/')[0]
|
||||
|
||||
|
||||
def principal_unit():
|
||||
"""Returns the principal unit of this unit, otherwise None"""
|
||||
# Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT
|
||||
principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None)
|
||||
# If it's empty, then this unit is the principal
|
||||
if principal_unit == '':
|
||||
return os.environ['JUJU_UNIT_NAME']
|
||||
elif principal_unit is not None:
|
||||
return principal_unit
|
||||
# For Juju 2.1 and below, let's try work out the principle unit by
|
||||
# the various charms' metadata.yaml.
|
||||
for reltype in relation_types():
|
||||
for rid in relation_ids(reltype):
|
||||
for unit in related_units(rid):
|
||||
md = _metadata_unit(unit)
|
||||
subordinate = md.pop('subordinate', None)
|
||||
if not subordinate:
|
||||
return unit
|
||||
return None
|
||||
|
||||
|
||||
@cached
|
||||
def remote_service_name(relid=None):
|
||||
"""The remote service name for a given relation-id (or the current relation)"""
|
||||
@ -478,6 +499,21 @@ def metadata():
|
||||
return yaml.safe_load(md)
|
||||
|
||||
|
||||
def _metadata_unit(unit):
|
||||
"""Given the name of a unit (e.g. apache2/0), get the unit charm's
|
||||
metadata.yaml. Very similar to metadata() but allows us to inspect
|
||||
other units. Unit needs to be co-located, such as a subordinate or
|
||||
principal/primary.
|
||||
|
||||
:returns: metadata.yaml as a python object.
|
||||
|
||||
"""
|
||||
basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
|
||||
unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
|
||||
with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
|
||||
return yaml.safe_load(md)
|
||||
|
||||
|
||||
@cached
|
||||
def relation_types():
|
||||
"""Get a list of relation types supported by this charm"""
|
||||
|
||||
@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer:
|
||||
https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
|
||||
"""
|
||||
import subprocess
|
||||
from os import environ
|
||||
import os
|
||||
from time import sleep
|
||||
from charmhelpers.core.hookenv import log
|
||||
|
||||
__author__ = 'Joseph Borg <joseph.borg@canonical.com>'
|
||||
|
||||
SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
|
||||
# The return code for "couldn't acquire lock" in Snap
|
||||
# (hopefully this will be improved).
|
||||
SNAP_NO_LOCK = 1
|
||||
SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks.
|
||||
SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
|
||||
SNAP_CHANNELS = [
|
||||
'edge',
|
||||
'beta',
|
||||
'candidate',
|
||||
'stable',
|
||||
]
|
||||
|
||||
|
||||
class CouldNotAcquireLockException(Exception):
|
||||
@ -47,13 +55,17 @@ def _snap_exec(commands):
|
||||
|
||||
while return_code is None or return_code == SNAP_NO_LOCK:
|
||||
try:
|
||||
return_code = subprocess.check_call(['snap'] + commands, env=environ)
|
||||
return_code = subprocess.check_call(['snap'] + commands,
|
||||
env=os.environ)
|
||||
except subprocess.CalledProcessError as e:
|
||||
retry_count += + 1
|
||||
if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
|
||||
raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
|
||||
raise CouldNotAcquireLockException(
|
||||
'Could not aquire lock after {} attempts'
|
||||
.format(SNAP_NO_LOCK_RETRY_COUNT))
|
||||
return_code = e.returncode
|
||||
log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
|
||||
log('Snap failed to acquire lock, trying again in {} seconds.'
|
||||
.format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN'))
|
||||
sleep(SNAP_NO_LOCK_RETRY_DELAY)
|
||||
|
||||
return return_code
|
||||
|
||||
@ -139,7 +139,7 @@ CLOUD_ARCHIVE_POCKETS = {
|
||||
'xenial-updates/ocata': 'xenial-updates/ocata',
|
||||
'ocata/proposed': 'xenial-proposed/ocata',
|
||||
'xenial-ocata/proposed': 'xenial-proposed/ocata',
|
||||
'xenial-ocata/newton': 'xenial-proposed/ocata',
|
||||
'xenial-proposed/ocata': 'xenial-proposed/ocata',
|
||||
# Pike
|
||||
'pike': 'xenial-updates/pike',
|
||||
'xenial-pike': 'xenial-updates/pike',
|
||||
@ -147,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = {
|
||||
'xenial-updates/pike': 'xenial-updates/pike',
|
||||
'pike/proposed': 'xenial-proposed/pike',
|
||||
'xenial-pike/proposed': 'xenial-proposed/pike',
|
||||
'xenial-pike/newton': 'xenial-proposed/pike',
|
||||
'xenial-proposed/pike': 'xenial-proposed/pike',
|
||||
# Queens
|
||||
'queens': 'xenial-updates/queens',
|
||||
'xenial-queens': 'xenial-updates/queens',
|
||||
@ -155,13 +155,13 @@ CLOUD_ARCHIVE_POCKETS = {
|
||||
'xenial-updates/queens': 'xenial-updates/queens',
|
||||
'queens/proposed': 'xenial-proposed/queens',
|
||||
'xenial-queens/proposed': 'xenial-proposed/queens',
|
||||
'xenial-queens/newton': 'xenial-proposed/queens',
|
||||
'xenial-proposed/queens': 'xenial-proposed/queens',
|
||||
}
|
||||
|
||||
|
||||
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
|
||||
CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries.
|
||||
CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times.
|
||||
CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times.
|
||||
|
||||
|
||||
def filter_installed_packages(packages):
|
||||
@ -364,6 +364,7 @@ def add_source(source, key=None, fail_invalid=False):
|
||||
(r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
|
||||
(r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
|
||||
(r"^cloud:(.*)$", _add_cloud_pocket),
|
||||
(r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check),
|
||||
])
|
||||
if source is None:
|
||||
source = ''
|
||||
|
||||
Loading…
Reference in New Issue
Block a user