openstack/charm-ceph-osd
Code Issues Proposed changes

Sync charm-helpers

Browse Source 
Change-Id: I12b0ba1b814cbba2dbb3474de5c0b180df03628a
This commit is contained in:
Ryan Beisner
2017-08-24 16:47:27 -05:00
parent ea59c761a4
commit 4189b4f8ee
12 changed files with 469 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hooks/charmhelpers/contrib/charmsupport/nrpe.py
View File

@@ -125,7 +125,7 @@ class CheckException(Exception):
class Check(object):
shortname_re = '[A-Za-z0-9-_]+$'
shortname_re = '[A-Za-z0-9-_.]+$'
service_template = ("""
#---------------------------------------------------
# This file is Juju managed

5
hooks/charmhelpers/contrib/hardening/apache/checks/config.py
View File

@@ -46,8 +46,9 @@ def get_audits():
context = ApacheConfContext()
settings = utils.get_settings('apache')
audits = [
FilePermissionAudit(paths='/etc/apache2/apache2.conf', user='root',
group='root', mode=0o0640),
FilePermissionAudit(paths=os.path.join(
settings['common']['apache_dir'], 'apache2.conf'),
user='root', group='root', mode=0o0640),
TemplatedFile(os.path.join(settings['common']['apache_dir'],
'mods-available/alias.conf'),

96
hooks/charmhelpers/contrib/openstack/context.py
View File

@@ -41,9 +41,9 @@ from charmhelpers.core.hookenv import (
charm_name,
DEBUG,
INFO,
WARNING,
ERROR,
status_set,
network_get_primary_address
)
from charmhelpers.core.sysctl import create as sysctl_create
@@ -80,6 +80,9 @@ from charmhelpers.contrib.openstack.neutron import (
from charmhelpers.contrib.openstack.ip import (
resolve_address,
INTERNAL,
ADMIN,
PUBLIC,
ADDRESS_MAP,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
@@ -87,7 +90,6 @@ from charmhelpers.contrib.network.ip import (
get_ipv6_addr,
get_netmask_for_address,
format_ipv6_addr,
is_address_in_network,
is_bridge_member,
is_ipv6_disabled,
)
@@ -97,6 +99,7 @@ from charmhelpers.contrib.openstack.utils import (
git_determine_usr_bin,
git_determine_python_path,
enable_memcache,
snap_install_requested,
)
from charmhelpers.core.unitdata import kv
@@ -244,6 +247,11 @@ class SharedDBContext(OSContextGenerator):
'database_password': rdata.get(password_setting),
'database_type': 'mysql'
}
# Note(coreycb): We can drop mysql+pymysql if we want when the
# following review lands, though it seems mysql+pymysql would
# be preferred. https://review.openstack.org/#/c/462190/
if snap_install_requested():
ctxt['database_type'] = 'mysql+pymysql'
if self.context_complete(ctxt):
db_ssl(rdata, ctxt, self.ssl_dir)
return ctxt
@@ -510,6 +518,10 @@ class CephContext(OSContextGenerator):
ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
if not ctxt.get('key'):
ctxt['key'] = relation_get('key', rid=rid, unit=unit)
if not ctxt.get('rbd_features'):
default_features = relation_get('rbd-features', rid=rid, unit=unit)
if default_features is not None:
ctxt['rbd_features'] = default_features
ceph_addrs = relation_get('ceph-public-address', rid=rid,
unit=unit)
@@ -610,7 +622,6 @@ class HAProxyContext(OSContextGenerator):
ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout')
if config('prefer-ipv6'):
ctxt['ipv6'] = True
ctxt['local_host'] = 'ip6-localhost'
ctxt['haproxy_host'] = '::'
else:
@@ -726,11 +737,17 @@ class ApacheSSLContext(OSContextGenerator):
return sorted(list(set(cns)))
def get_network_addresses(self):
"""For each network configured, return corresponding address and vip
(if available).
"""For each network configured, return corresponding address and
hostnamr or vip (if available).
Returns a list of tuples of the form:
[(address_in_net_a, hostname_in_net_a),
(address_in_net_b, hostname_in_net_b),
...]
or, if no hostnames(s) available:
[(address_in_net_a, vip_in_net_a),
(address_in_net_b, vip_in_net_b),
...]
@@ -742,32 +759,27 @@ class ApacheSSLContext(OSContextGenerator):
...]
"""
addresses = []
if config('vip'):
vips = config('vip').split()
else:
vips = []
for net_type in ['os-internal-network', 'os-admin-network',
'os-public-network']:
addr = get_address_in_network(config(net_type),
unit_get('private-address'))
if len(vips) > 1 and is_clustered():
if not config(net_type):
log("Multiple networks configured but net_type "
"is None (%s)." % net_type, level=WARNING)
continue
for vip in vips:
if is_address_in_network(config(net_type), vip):
addresses.append((addr, vip))
break
elif is_clustered() and config('vip'):
addresses.append((addr, config('vip')))
for net_type in [INTERNAL, ADMIN, PUBLIC]:
net_config = config(ADDRESS_MAP[net_type]['config'])
# NOTE(jamespage): Fallback must always be private address
# as this is used to bind services on the
# local unit.
fallback = unit_get("private-address")
if net_config:
addr = get_address_in_network(net_config,
fallback)
else:
addresses.append((addr, addr))
try:
addr = network_get_primary_address(
ADDRESS_MAP[net_type]['binding']
)
except NotImplementedError:
addr = fallback
return sorted(addresses)
endpoint = resolve_address(net_type)
addresses.append((addr, endpoint))
return sorted(set(addresses))
def __call__(self):
if isinstance(self.external_ports, six.string_types):
@@ -794,7 +806,7 @@ class ApacheSSLContext(OSContextGenerator):
self.configure_cert(cn)
addresses = self.get_network_addresses()
for address, endpoint in sorted(set(addresses)):
for address, endpoint in addresses:
for api_port in self.external_ports:
ext_port = determine_apache_port(api_port,
singlenode_mode=True)
@@ -1397,14 +1409,38 @@ class NeutronAPIContext(OSContextGenerator):
'rel_key': 'dns-domain',
'default': None,
},
'polling_interval': {
'rel_key': 'polling-interval',
'default': 2,
},
'rpc_response_timeout': {
'rel_key': 'rpc-response-timeout',
'default': 60,
},
'report_interval': {
'rel_key': 'report-interval',
'default': 30,
},
'enable_qos': {
'rel_key': 'enable-qos',
'default': False,
},
}
ctxt = self.get_neutron_options({})
for rid in relation_ids('neutron-plugin-api'):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
# The l2-population key is used by the context as a way of
# checking if the api service on the other end is sending data
# in a recent format.
if 'l2-population' in rdata:
ctxt.update(self.get_neutron_options(rdata))
if ctxt['enable_qos']:
ctxt['extension_drivers'] = 'qos'
else:
ctxt['extension_drivers'] = ''
return ctxt
def get_neutron_options(self, rdata):

135
hooks/charmhelpers/contrib/openstack/utils.py
View File

@@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import (
status_set,
hook_name,
application_version_set,
cached,
)
from charmhelpers.core.strutils import BasicStringComparator
@@ -90,6 +91,13 @@ from charmhelpers.fetch import (
GPGKeyError,
get_upstream_version
)
from charmhelpers.fetch.snap import (
snap_install,
snap_refresh,
SNAP_CHANNELS,
)
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
from charmhelpers.contrib.openstack.exceptions import OSContextError
@@ -178,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([
('ocata',
['2.11.0', '2.12.0', '2.13.0']),
('pike',
['2.13.0']),
['2.13.0', '2.15.0']),
])
# >= Liberty version->codename mapping
@@ -327,8 +335,10 @@ def get_os_codename_install_source(src):
return ca_rel
# Best guess match based on deb string provided
if src.startswith('deb') or src.startswith('ppa'):
for k, v in six.iteritems(OPENSTACK_CODENAMES):
if (src.startswith('deb') or
src.startswith('ppa') or
src.startswith('snap')):
for v in OPENSTACK_CODENAMES.values():
if v in src:
return v
@@ -397,6 +407,19 @@ def get_swift_codename(version):
def get_os_codename_package(package, fatal=True):
'''Derive OpenStack release codename from an installed package.'''
if snap_install_requested():
cmd = ['snap', 'list', package]
try:
out = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
return None
lines = out.split('\n')
for line in lines:
if package in line:
# Second item in list is Version
return line.split()[1]
import apt_pkg as apt
cache = apt_cache()
@@ -613,6 +636,9 @@ def openstack_upgrade_available(package):
import apt_pkg as apt
src = config('openstack-origin')
cur_vers = get_os_version_package(package)
if not cur_vers:
# The package has not been installed yet do not attempt upgrade
return False
if "swift" in package:
codename = get_os_codename_install_source(src)
avail_vers = get_os_version_codename_swift(codename)
@@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False,
return wrap
def ordered(orderme):
"""Converts the provided dictionary into a collections.OrderedDict.
The items in the returned OrderedDict will be inserted based on the
natural sort order of the keys. Nested dictionaries will also be sorted
in order to ensure fully predictable ordering.
:param orderme: the dict to order
:return: collections.OrderedDict
:raises: ValueError: if `orderme` isn't a dict instance.
"""
if not isinstance(orderme, dict):
raise ValueError('argument must be a dict type')
result = OrderedDict()
for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]):
if isinstance(v, dict):
result[k] = ordered(v)
else:
result[k] = v
return result
def config_flags_parser(config_flags):
"""Parses config flags string into dict.
@@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags):
example, a string in the format of 'key1=value1, key2=value2' will
return a dict of:
{'key1': 'value1',
'key2': 'value2'}.
{'key1': 'value1', 'key2': 'value2'}.
2. A string in the above format, but supporting a comma-delimited list
of values for the same key. For example, a string in the format of
'key1=value1, key2=value3,value4,value5' will return a dict of:
{'key1', 'value1',
'key2', 'value2,value3,value4'}
{'key1': 'value1', 'key2': 'value2,value3,value4'}
3. A string containing a colon character (:) prior to an equal
character (=) will be treated as yaml and parsed as such. This can be
@@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags):
equals = config_flags.find('=')
if colon > 0:
if colon < equals or equals < 0:
return yaml.safe_load(config_flags)
return ordered(yaml.safe_load(config_flags))
if config_flags.find('==') >= 0:
juju_log("config_flags is not in expected format (key=value)",
@@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags):
# split on '='.
split = config_flags.strip(' =').split('=')
limit = len(split)
flags = {}
flags = OrderedDict()
for i in range(0, limit - 1):
current = split[i]
next = split[i + 1]
@@ -1994,3 +2042,72 @@ def update_json_file(filename, items):
policy.update(items)
with open(filename, "w") as fd:
fd.write(json.dumps(policy, indent=4))
@cached
def snap_install_requested():
""" Determine if installing from snaps
If openstack-origin is of the form snap:channel-series-release
and channel is in SNAPS_CHANNELS return True.
"""
origin = config('openstack-origin') or ""
if not origin.startswith('snap:'):
return False
_src = origin[5:]
channel, series, release = _src.split('-')
if channel.lower() in SNAP_CHANNELS:
return True
return False
def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
"""Generate a dictionary of snap install information from origin
@param snaps: List of snaps
@param src: String of openstack-origin or source of the form
snap:channel-series-track
@param mode: String classic, devmode or jailmode
@returns: Dictionary of snaps with channels and modes
"""
if not src.startswith('snap:'):
juju_log("Snap source is not a snap origin", 'WARN')
return {}
_src = src[5:]
_channel, _series, _release = _src.split('-')
channel = '--channel={}/{}'.format(_release, _channel)
return {snap: {'channel': channel, 'mode': mode}
for snap in snaps}
def install_os_snaps(snaps, refresh=False):
"""Install OpenStack snaps from channel and with mode
@param snaps: Dictionary of snaps with channels and modes of the form:
{'snap_name': {'channel': 'snap_channel',
'mode': 'snap_mode'}}
Where channel a snapstore channel and mode is --classic, --devmode or
--jailmode.
@param post_snap_install: Callback function to run after snaps have been
installed
"""
def _ensure_flag(flag):
if flag.startswith('--'):
return flag
return '--{}'.format(flag)
if refresh:
for snap in snaps.keys():
snap_refresh(snap,
_ensure_flag(snaps[snap]['channel']),
_ensure_flag(snaps[snap]['mode']))
else:
for snap in snaps.keys():
snap_install(snap,
_ensure_flag(snaps[snap]['channel']),
_ensure_flag(snaps[snap]['mode']))

74
hooks/charmhelpers/contrib/storage/linux/bcache.py Normal file
View File

@@ -0,0 +1,74 @@
# Copyright 2017 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import json
from charmhelpers.core.hookenv import log
stats_intervals = ['stats_day', 'stats_five_minute',
'stats_hour', 'stats_total']
SYSFS = '/sys'
class Bcache(object):
"""Bcache behaviour
"""
def __init__(self, cachepath):
self.cachepath = cachepath
@classmethod
def fromdevice(cls, devname):
return cls('{}/block/{}/bcache'.format(SYSFS, devname))
def __str__(self):
return self.cachepath
def get_stats(self, interval):
"""Get cache stats
"""
intervaldir = 'stats_{}'.format(interval)
path = "{}/{}".format(self.cachepath, intervaldir)
out = dict()
for elem in os.listdir(path):
out[elem] = open('{}/{}'.format(path, elem)).read().strip()
return out
def get_bcache_fs():
"""Return all cache sets
"""
cachesetroot = "{}/fs/bcache".format(SYSFS)
try:
dirs = os.listdir(cachesetroot)
except OSError:
log("No bcache fs found")
return []
cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')])
return cacheset
def get_stats_action(cachespec, interval):
"""Action for getting bcache statistics for a given cachespec.
Cachespec can either be a device name, eg. 'sdb', which will retrieve
cache stats for the given device, or 'global', which will retrieve stats
for all cachesets
"""
if cachespec == 'global':
caches = get_bcache_fs()
else:
caches = [Bcache.fromdevice(cachespec)]
res = dict((c.cachepath, c.get_stats(interval)) for c in caches)
return json.dumps(res, indent=4, separators=(',', ': '))

2
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

@@ -1372,7 +1372,7 @@ class CephConfContext(object):
return {}
conf = config_flags_parser(conf)
if type(conf) != dict:
if not isinstance(conf, dict):
log("Provided config-flags is not a dictionary - ignoring",
level=WARNING)
return {}