From 4d48338c9647cb506bbfb6a18dba5694ee051ad2 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 12 Jul 2017 15:57:54 +0100 Subject: [PATCH] Update notification config >= mitaka Use oslo_messaging_notifications for mitaka or later releases including setting the transport_url to the value provided by the AMQP context. This removes use of deprecated configuration options for ceilometer notifications. This change includes some refactoring to allow the topics to use for notifications to be configured specifically for this charm; future changes can use this to enable/disable designate notifications dynamically. Also includes redux of services check for amulet tests to drop all checks apart from those for the neutron-api units. Change-Id: Ib66371c0c479e0b341055941842e43ac57d4151d --- .../charmhelpers/contrib/charmsupport/nrpe.py | 2 +- .../charmhelpers/contrib/openstack/context.py | 48 +++++++-- .../contrib/openstack/templates/ceph.conf | 5 +- .../templates/section-oslo-notifications | 8 ++ hooks/charmhelpers/contrib/openstack/utils.py | 101 +++++++++++++++++- .../contrib/storage/linux/bcache.py | 74 +++++++++++++ .../contrib/storage/linux/ceph.py | 2 +- hooks/charmhelpers/core/hookenv.py | 39 +++++++ hooks/charmhelpers/fetch/snap.py | 22 +++- hooks/charmhelpers/fetch/ubuntu.py | 9 +- hooks/neutron_api_context.py | 19 ++++ hooks/neutron_api_utils.py | 2 +- templates/mitaka/neutron.conf | 4 +- templates/newton/neutron.conf | 4 +- templates/pike/neutron.conf | 4 +- tests/charmhelpers/__init__.py | 61 +++++++++++ .../contrib/openstack/amulet/utils.py | 101 ++++++++++++------ tests/charmhelpers/core/hookenv.py | 39 +++++++ tests/charmhelpers/core/host.py | 2 + 19 files changed, 482 insertions(+), 64 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications create mode 100644 hooks/charmhelpers/contrib/storage/linux/bcache.py diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 424b7f76..80d574dc 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -125,7 +125,7 @@ class CheckException(Exception): class Check(object): - shortname_re = '[A-Za-z0-9-_]+$' + shortname_re = '[A-Za-z0-9-_.]+$' service_template = (""" #--------------------------------------------------- # This file is Juju managed diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 3c313c1a..e9543638 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -97,6 +97,7 @@ from charmhelpers.contrib.openstack.utils import ( git_determine_usr_bin, git_determine_python_path, enable_memcache, + snap_install_requested, ) from charmhelpers.core.unitdata import kv @@ -244,6 +245,11 @@ class SharedDBContext(OSContextGenerator): 'database_password': rdata.get(password_setting), 'database_type': 'mysql' } + # Note(coreycb): We can drop mysql+pymysql if we want when the + # following review lands, though it seems mysql+pymysql would + # be preferred. https://review.openstack.org/#/c/462190/ + if snap_install_requested(): + ctxt['database_type'] = 'mysql+pymysql' if self.context_complete(ctxt): db_ssl(rdata, ctxt, self.ssl_dir) return ctxt @@ -510,6 +516,10 @@ class CephContext(OSContextGenerator): ctxt['auth'] = relation_get('auth', rid=rid, unit=unit) if not ctxt.get('key'): ctxt['key'] = relation_get('key', rid=rid, unit=unit) + if not ctxt.get('rbd_features'): + default_features = relation_get('rbd-features', rid=rid, unit=unit) + if default_features is not None: + ctxt['rbd_features'] = default_features ceph_addrs = relation_get('ceph-public-address', rid=rid, unit=unit) @@ -726,11 +736,17 @@ class ApacheSSLContext(OSContextGenerator): return sorted(list(set(cns))) def get_network_addresses(self): - """For each network configured, return corresponding address and vip - (if available). + """For each network configured, return corresponding address and + hostnamr or vip (if available). Returns a list of tuples of the form: + [(address_in_net_a, hostname_in_net_a), + (address_in_net_b, hostname_in_net_b), + ...] + + or, if no hostnames(s) available: + [(address_in_net_a, vip_in_net_a), (address_in_net_b, vip_in_net_b), ...] @@ -747,18 +763,22 @@ class ApacheSSLContext(OSContextGenerator): else: vips = [] - for net_type in ['os-internal-network', 'os-admin-network', - 'os-public-network']: - addr = get_address_in_network(config(net_type), + for net_type in ['internal', 'admin', 'public']: + net_config = config('os-{}-network'.format(net_type)) + addr = get_address_in_network(net_config, unit_get('private-address')) - if len(vips) > 1 and is_clustered(): - if not config(net_type): + + hostname_config = config('os-{}-hostname'.format(net_type)) + if hostname_config: + addresses.append((addr, hostname_config)) + elif len(vips) > 1 and is_clustered(): + if not net_config: log("Multiple networks configured but net_type " "is None (%s)." % net_type, level=WARNING) continue for vip in vips: - if is_address_in_network(config(net_type), vip): + if is_address_in_network(net_config, vip): addresses.append((addr, vip)) break @@ -1409,14 +1429,26 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'report-interval', 'default': 30, }, + 'enable_qos': { + 'rel_key': 'enable-qos', + 'default': False, + }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) + # The l2-population key is used by the context as a way of + # checking if the api service on the other end is sending data + # in a recent format. if 'l2-population' in rdata: ctxt.update(self.get_neutron_options(rdata)) + if ctxt['enable_qos']: + ctxt['extension_drivers'] = 'qos' + else: + ctxt['extension_drivers'] = '' + return ctxt def get_neutron_options(self, rdata): diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 33ceee25..ed5c4f10 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -1,6 +1,6 @@ ############################################################################### # [ WARNING ] -# cinder configuration file maintained by Juju +# ceph configuration file maintained by Juju # local changes may be overwritten. ############################################################################### [global] @@ -12,6 +12,9 @@ mon host = {{ mon_hosts }} log to syslog = {{ use_syslog }} err to syslog = {{ use_syslog }} clog to syslog = {{ use_syslog }} +{% if rbd_features %} +rbd default features = {{ rbd_features }} +{% endif %} [client] {% if rbd_client_cache_settings -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications new file mode 100644 index 00000000..5dccd4bb --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications @@ -0,0 +1,8 @@ +{% if transport_url -%} +[oslo_messaging_notifications] +driver = messagingv2 +transport_url = {{ transport_url }} +{% if notification_topics -%} +topics = {{ notification_topics }} +{% endif -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 575d4b83..837a1674 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import ( status_set, hook_name, application_version_set, + cached, ) from charmhelpers.core.strutils import BasicStringComparator @@ -90,6 +91,13 @@ from charmhelpers.fetch import ( GPGKeyError, get_upstream_version ) + +from charmhelpers.fetch.snap import ( + snap_install, + snap_refresh, + SNAP_CHANNELS, +) + from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device from charmhelpers.contrib.openstack.exceptions import OSContextError @@ -178,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([ ('ocata', ['2.11.0', '2.12.0', '2.13.0']), ('pike', - ['2.13.0']), + ['2.13.0', '2.15.0']), ]) # >= Liberty version->codename mapping @@ -327,8 +335,10 @@ def get_os_codename_install_source(src): return ca_rel # Best guess match based on deb string provided - if src.startswith('deb') or src.startswith('ppa'): - for k, v in six.iteritems(OPENSTACK_CODENAMES): + if (src.startswith('deb') or + src.startswith('ppa') or + src.startswith('snap')): + for v in OPENSTACK_CODENAMES.values(): if v in src: return v @@ -397,6 +407,19 @@ def get_swift_codename(version): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + + if snap_install_requested(): + cmd = ['snap', 'list', package] + try: + out = subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + return None + lines = out.split('\n') + for line in lines: + if package in line: + # Second item in list is Version + return line.split()[1] + import apt_pkg as apt cache = apt_cache() @@ -613,6 +636,9 @@ def openstack_upgrade_available(package): import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) + if not cur_vers: + # The package has not been installed yet do not attempt upgrade + return False if "swift" in package: codename = get_os_codename_install_source(src) avail_vers = get_os_version_codename_swift(codename) @@ -2016,3 +2042,72 @@ def update_json_file(filename, items): policy.update(items) with open(filename, "w") as fd: fd.write(json.dumps(policy, indent=4)) + + +@cached +def snap_install_requested(): + """ Determine if installing from snaps + + If openstack-origin is of the form snap:channel-series-release + and channel is in SNAPS_CHANNELS return True. + """ + origin = config('openstack-origin') or "" + if not origin.startswith('snap:'): + return False + + _src = origin[5:] + channel, series, release = _src.split('-') + if channel.lower() in SNAP_CHANNELS: + return True + return False + + +def get_snaps_install_info_from_origin(snaps, src, mode='classic'): + """Generate a dictionary of snap install information from origin + + @param snaps: List of snaps + @param src: String of openstack-origin or source of the form + snap:channel-series-track + @param mode: String classic, devmode or jailmode + @returns: Dictionary of snaps with channels and modes + """ + + if not src.startswith('snap:'): + juju_log("Snap source is not a snap origin", 'WARN') + return {} + + _src = src[5:] + _channel, _series, _release = _src.split('-') + channel = '--channel={}/{}'.format(_release, _channel) + + return {snap: {'channel': channel, 'mode': mode} + for snap in snaps} + + +def install_os_snaps(snaps, refresh=False): + """Install OpenStack snaps from channel and with mode + + @param snaps: Dictionary of snaps with channels and modes of the form: + {'snap_name': {'channel': 'snap_channel', + 'mode': 'snap_mode'}} + Where channel a snapstore channel and mode is --classic, --devmode or + --jailmode. + @param post_snap_install: Callback function to run after snaps have been + installed + """ + + def _ensure_flag(flag): + if flag.startswith('--'): + return flag + return '--{}'.format(flag) + + if refresh: + for snap in snaps.keys(): + snap_refresh(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) + else: + for snap in snaps.keys(): + snap_install(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) diff --git a/hooks/charmhelpers/contrib/storage/linux/bcache.py b/hooks/charmhelpers/contrib/storage/linux/bcache.py new file mode 100644 index 00000000..605991e1 --- /dev/null +++ b/hooks/charmhelpers/contrib/storage/linux/bcache.py @@ -0,0 +1,74 @@ +# Copyright 2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import json + +from charmhelpers.core.hookenv import log + +stats_intervals = ['stats_day', 'stats_five_minute', + 'stats_hour', 'stats_total'] + +SYSFS = '/sys' + + +class Bcache(object): + """Bcache behaviour + """ + + def __init__(self, cachepath): + self.cachepath = cachepath + + @classmethod + def fromdevice(cls, devname): + return cls('{}/block/{}/bcache'.format(SYSFS, devname)) + + def __str__(self): + return self.cachepath + + def get_stats(self, interval): + """Get cache stats + """ + intervaldir = 'stats_{}'.format(interval) + path = "{}/{}".format(self.cachepath, intervaldir) + out = dict() + for elem in os.listdir(path): + out[elem] = open('{}/{}'.format(path, elem)).read().strip() + return out + + +def get_bcache_fs(): + """Return all cache sets + """ + cachesetroot = "{}/fs/bcache".format(SYSFS) + try: + dirs = os.listdir(cachesetroot) + except OSError: + log("No bcache fs found") + return [] + cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')]) + return cacheset + + +def get_stats_action(cachespec, interval): + """Action for getting bcache statistics for a given cachespec. + Cachespec can either be a device name, eg. 'sdb', which will retrieve + cache stats for the given device, or 'global', which will retrieve stats + for all cachesets + """ + if cachespec == 'global': + caches = get_bcache_fs() + else: + caches = [Bcache.fromdevice(cachespec)] + res = dict((c.cachepath, c.get_stats(interval)) for c in caches) + return json.dumps(res, indent=4, separators=(',', ': ')) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 1f0540a1..e5a01b1b 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -1372,7 +1372,7 @@ class CephConfContext(object): return {} conf = config_flags_parser(conf) - if type(conf) != dict: + if not isinstance(conf, dict): log("Provided config-flags is not a dictionary - ignoring", level=WARNING) return {} diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index e44e22bf..814a9354 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -202,6 +202,27 @@ def service_name(): return local_unit().split('/')[0] +def principal_unit(): + """Returns the principal unit of this unit, otherwise None""" + # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT + principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None) + # If it's empty, then this unit is the principal + if principal_unit == '': + return os.environ['JUJU_UNIT_NAME'] + elif principal_unit is not None: + return principal_unit + # For Juju 2.1 and below, let's try work out the principle unit by + # the various charms' metadata.yaml. + for reltype in relation_types(): + for rid in relation_ids(reltype): + for unit in related_units(rid): + md = _metadata_unit(unit) + subordinate = md.pop('subordinate', None) + if not subordinate: + return unit + return None + + @cached def remote_service_name(relid=None): """The remote service name for a given relation-id (or the current relation)""" @@ -478,6 +499,21 @@ def metadata(): return yaml.safe_load(md) +def _metadata_unit(unit): + """Given the name of a unit (e.g. apache2/0), get the unit charm's + metadata.yaml. Very similar to metadata() but allows us to inspect + other units. Unit needs to be co-located, such as a subordinate or + principal/primary. + + :returns: metadata.yaml as a python object. + + """ + basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) + unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) + with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md: + return yaml.safe_load(md) + + @cached def relation_types(): """Get a list of relation types supported by this charm""" @@ -753,6 +789,9 @@ class Hooks(object): def charm_dir(): """Return the root directory of the current charm""" + d = os.environ.get('JUJU_CHARM_DIR') + if d is not None: + return d return os.environ.get('CHARM_DIR') diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py index 23c707b0..112a54c3 100644 --- a/hooks/charmhelpers/fetch/snap.py +++ b/hooks/charmhelpers/fetch/snap.py @@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer: https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html """ import subprocess -from os import environ +import os from time import sleep from charmhelpers.core.hookenv import log __author__ = 'Joseph Borg ' -SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +# The return code for "couldn't acquire lock" in Snap +# (hopefully this will be improved). +SNAP_NO_LOCK = 1 SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +SNAP_CHANNELS = [ + 'edge', + 'beta', + 'candidate', + 'stable', +] class CouldNotAcquireLockException(Exception): @@ -47,13 +55,17 @@ def _snap_exec(commands): while return_code is None or return_code == SNAP_NO_LOCK: try: - return_code = subprocess.check_call(['snap'] + commands, env=environ) + return_code = subprocess.check_call(['snap'] + commands, + env=os.environ) except subprocess.CalledProcessError as e: retry_count += + 1 if retry_count > SNAP_NO_LOCK_RETRY_COUNT: - raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + raise CouldNotAcquireLockException( + 'Could not aquire lock after {} attempts' + .format(SNAP_NO_LOCK_RETRY_COUNT)) return_code = e.returncode - log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + log('Snap failed to acquire lock, trying again in {} seconds.' + .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN')) sleep(SNAP_NO_LOCK_RETRY_DELAY) return return_code diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 57b5fb61..545348ff 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -139,7 +139,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/ocata': 'xenial-updates/ocata', 'ocata/proposed': 'xenial-proposed/ocata', 'xenial-ocata/proposed': 'xenial-proposed/ocata', - 'xenial-ocata/newton': 'xenial-proposed/ocata', + 'xenial-proposed/ocata': 'xenial-proposed/ocata', # Pike 'pike': 'xenial-updates/pike', 'xenial-pike': 'xenial-updates/pike', @@ -147,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/pike': 'xenial-updates/pike', 'pike/proposed': 'xenial-proposed/pike', 'xenial-pike/proposed': 'xenial-proposed/pike', - 'xenial-pike/newton': 'xenial-proposed/pike', + 'xenial-proposed/pike': 'xenial-proposed/pike', # Queens 'queens': 'xenial-updates/queens', 'xenial-queens': 'xenial-updates/queens', @@ -155,13 +155,13 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/queens': 'xenial-updates/queens', 'queens/proposed': 'xenial-proposed/queens', 'xenial-queens/proposed': 'xenial-proposed/queens', - 'xenial-queens/newton': 'xenial-proposed/queens', + 'xenial-proposed/queens': 'xenial-proposed/queens', } APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. +CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -364,6 +364,7 @@ def add_source(source, key=None, fail_invalid=False): (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), (r"^cloud:(.*)$", _add_cloud_pocket), + (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), ]) if source is None: source = '' diff --git a/hooks/neutron_api_context.py b/hooks/neutron_api_context.py index 62074f59..f4e2beb8 100644 --- a/hooks/neutron_api_context.py +++ b/hooks/neutron_api_context.py @@ -48,6 +48,13 @@ TENANT_NET_TYPES = [VXLAN, GRE, VLAN, FLAT, LOCAL] EXTENSION_DRIVER_PORT_SECURITY = 'port_security' EXTENSION_DRIVER_DNS = 'dns' +ETC_NEUTRON = '/etc/neutron' + +NOTIFICATION_TOPICS = [ + 'notifications', + 'notifications_designate' +] + # Domain name validation regex which is used to certify that # the domain-name consists only of valid characters, is not # longer than 63 characters in length for any name segment, @@ -626,3 +633,15 @@ class MidonetContext(context.OSContextGenerator): if self.context_complete(ctxt): return ctxt return {} + + +class NeutronAMQPContext(context.AMQPContext): + '''AMQP context with Neutron API sauce''' + + def __init__(self): + super(NeutronAMQPContext, self).__init__(ssl_dir=ETC_NEUTRON) + + def __call__(self): + context = super(NeutronAMQPContext, self).__call__() + context['notification_topics'] = ','.join(NOTIFICATION_TOPICS) + return context diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index b0b5d891..c6e64d27 100755 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -169,7 +169,7 @@ ML2_SRIOV_INI = os.path.join(NEUTRON_CONF_DIR, BASE_RESOURCE_MAP = OrderedDict([ (NEUTRON_CONF, { 'services': ['neutron-server'], - 'contexts': [context.AMQPContext(ssl_dir=NEUTRON_CONF_DIR), + 'contexts': [neutron_api_context.NeutronAMQPContext(), context.SharedDBContext( user=config('database-user'), database=config('database'), diff --git a/templates/mitaka/neutron.conf b/templates/mitaka/neutron.conf index ffbee1d3..45f13371 100644 --- a/templates/mitaka/neutron.conf +++ b/templates/mitaka/neutron.conf @@ -11,8 +11,6 @@ use_syslog = {{ use_syslog }} state_path = /var/lib/neutron bind_host = {{ bind_host }} auth_strategy = keystone -notification_driver = messaging -notification_topics = notifications,notifications_designate api_workers = {{ workers }} rpc_workers = {{ workers }} @@ -109,6 +107,8 @@ root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf {% include "section-rabbitmq-oslo" %} +{% include "section-oslo-notifications" %} + [oslo_concurrency] lock_path = $state_path/lock diff --git a/templates/newton/neutron.conf b/templates/newton/neutron.conf index fb0aa89d..387cdcd2 100644 --- a/templates/newton/neutron.conf +++ b/templates/newton/neutron.conf @@ -11,8 +11,6 @@ use_syslog = {{ use_syslog }} state_path = /var/lib/neutron bind_host = {{ bind_host }} auth_strategy = keystone -notification_driver = messaging -notification_topics = notifications,notifications_designate api_workers = {{ workers }} rpc_workers = {{ workers }} @@ -109,6 +107,8 @@ root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf {% include "section-rabbitmq-oslo" %} +{% include "section-oslo-notifications" %} + [oslo_concurrency] lock_path = $state_path/lock diff --git a/templates/pike/neutron.conf b/templates/pike/neutron.conf index 410b556b..5612bf4b 100644 --- a/templates/pike/neutron.conf +++ b/templates/pike/neutron.conf @@ -11,8 +11,6 @@ use_syslog = {{ use_syslog }} state_path = /var/lib/neutron bind_host = {{ bind_host }} auth_strategy = keystone -notification_driver = messaging -notification_topics = notifications,notifications_designate api_workers = {{ workers }} rpc_workers = {{ workers }} @@ -105,6 +103,8 @@ root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf {% include "section-rabbitmq-oslo" %} +{% include "section-oslo-notifications" %} + [oslo_concurrency] lock_path = $state_path/lock diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py index 48867880..e7aa4715 100644 --- a/tests/charmhelpers/__init__.py +++ b/tests/charmhelpers/__init__.py @@ -14,6 +14,11 @@ # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. +from __future__ import print_function +from __future__ import absolute_import + +import functools +import inspect import subprocess import sys @@ -34,3 +39,59 @@ except ImportError: else: subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) import yaml # flake8: noqa + + +# Holds a list of mapping of mangled function names that have been deprecated +# using the @deprecate decorator below. This is so that the warning is only +# printed once for each usage of the function. +__deprecated_functions = {} + + +def deprecate(warning, date=None, log=None): + """Add a deprecation warning the first time the function is used. + The date, which is a string in semi-ISO8660 format indicate the year-month + that the function is officially going to be removed. + + usage: + + @deprecate('use core/fetch/add_source() instead', '2017-04') + def contributed_add_source_thing(...): + ... + + And it then prints to the log ONCE that the function is deprecated. + The reason for passing the logging function (log) is so that hookenv.log + can be used for a charm if needed. + + :param warning: String to indicat where it has moved ot. + :param date: optional sting, in YYYY-MM format to indicate when the + function will definitely (probably) be removed. + :param log: The log function to call to log. If not, logs to stdout + """ + def wrap(f): + + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + try: + module = inspect.getmodule(f) + file = inspect.getsourcefile(f) + lines = inspect.getsourcelines(f) + f_name = "{}-{}-{}..{}-{}".format( + module.__name__, file, lines[0], lines[-1], f.__name__) + except (IOError, TypeError): + # assume it was local, so just use the name of the function + f_name = f.__name__ + if f_name not in __deprecated_functions: + __deprecated_functions[f_name] = True + s = "DEPRECATION WARNING: Function {} is being removed".format( + f.__name__) + if date: + s = "{} on/around {}".format(s, date) + if warning: + s = "{} : {}".format(s, warning) + if log: + log(s) + else: + print(s) + return f(*args, **kwargs) + return wrapped_f + return wrap diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index bcef4cd0..c8edbf65 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -25,9 +25,12 @@ import urlparse import cinderclient.v1.client as cinder_client import glanceclient.v1.client as glance_client import heatclient.v1.client as heat_client -import keystoneclient.v2_0 as keystone_client -from keystoneclient.auth.identity import v3 as keystone_id_v3 -from keystoneclient import session as keystone_session +from keystoneclient.v2_0 import client as keystone_client +from keystoneauth1.identity import ( + v3, + v2, +) +from keystoneauth1 import session as keystone_session from keystoneclient.v3 import client as keystone_client_v3 from novaclient import exceptions @@ -368,12 +371,20 @@ class OpenStackAmuletUtils(AmuletUtils): port) if not api_version or api_version == 2: ep = base_ep + "/v2.0" - return keystone_client.Client(username=username, password=password, - tenant_name=project_name, - auth_url=ep) + auth = v2.Password( + username=username, + password=password, + tenant_name=project_name, + auth_url=ep + ) + sess = keystone_session.Session(auth=auth) + client = keystone_client.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client else: ep = base_ep + "/v3" - auth = keystone_id_v3.Password( + auth = v3.Password( user_domain_name=user_domain_name, username=username, password=password, @@ -382,36 +393,45 @@ class OpenStackAmuletUtils(AmuletUtils): project_name=project_name, auth_url=ep ) - return keystone_client_v3.Client( - session=keystone_session.Session(auth=auth) - ) + sess = keystone_session.Session(auth=auth) + client = keystone_client_v3.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client def authenticate_keystone_admin(self, keystone_sentry, user, password, tenant=None, api_version=None, - keystone_ip=None): + keystone_ip=None, user_domain_name=None, + project_domain_name=None, + project_name=None): """Authenticates admin user with the keystone admin endpoint.""" self.log.debug('Authenticating keystone admin...') if not keystone_ip: keystone_ip = keystone_sentry.info['public-address'] - user_domain_name = None - domain_name = None - if api_version == 3: + # To support backward compatibility usage of this function + if not project_name: + project_name = tenant + if api_version == 3 and not user_domain_name: user_domain_name = 'admin_domain' - domain_name = user_domain_name + if api_version == 3 and not project_domain_name: + project_domain_name = 'admin_domain' + if api_version == 3 and not project_name: + project_name = 'admin' - return self.authenticate_keystone(keystone_ip, user, password, - project_name=tenant, - api_version=api_version, - user_domain_name=user_domain_name, - domain_name=domain_name, - admin_port=True) + return self.authenticate_keystone( + keystone_ip, user, password, + api_version=api_version, + user_domain_name=user_domain_name, + project_domain_name=project_domain_name, + project_name=project_name, + admin_port=True) def authenticate_keystone_user(self, keystone, user, password, tenant): """Authenticates a regular user with the keystone public endpoint.""" self.log.debug('Authenticating keystone user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') keystone_ip = urlparse.urlparse(ep).hostname return self.authenticate_keystone(keystone_ip, user, password, @@ -421,22 +441,32 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates admin user with glance.""" self.log.debug('Authenticating glance admin...') ep = keystone.service_catalog.url_for(service_type='image', - endpoint_type='adminURL') - return glance_client.Client(ep, token=keystone.auth_token) + interface='adminURL') + if keystone.session: + return glance_client.Client(ep, session=keystone.session) + else: + return glance_client.Client(ep, token=keystone.auth_token) def authenticate_heat_admin(self, keystone): """Authenticates the admin user with heat.""" self.log.debug('Authenticating heat admin...') ep = keystone.service_catalog.url_for(service_type='orchestration', - endpoint_type='publicURL') - return heat_client.Client(endpoint=ep, token=keystone.auth_token) + interface='publicURL') + if keystone.session: + return heat_client.Client(endpoint=ep, session=keystone.session) + else: + return heat_client.Client(endpoint=ep, token=keystone.auth_token) def authenticate_nova_user(self, keystone, user, password, tenant): """Authenticates a regular user with nova-api.""" self.log.debug('Authenticating nova user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - if novaclient.__version__[0] >= "7": + interface='publicURL') + if keystone.session: + return nova_client.Client(NOVA_CLIENT_VERSION, + session=keystone.session, + auth_url=ep) + elif novaclient.__version__[0] >= "7": return nova_client.Client(NOVA_CLIENT_VERSION, username=user, password=password, project_name=tenant, auth_url=ep) @@ -449,12 +479,15 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates a regular user with swift api.""" self.log.debug('Authenticating swift user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - return swiftclient.Connection(authurl=ep, - user=user, - key=password, - tenant_name=tenant, - auth_version='2.0') + interface='publicURL') + if keystone.session: + return swiftclient.Connection(session=keystone.session) + else: + return swiftclient.Connection(authurl=ep, + user=user, + key=password, + tenant_name=tenant, + auth_version='2.0') def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto", ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True): diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index e44e22bf..814a9354 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -202,6 +202,27 @@ def service_name(): return local_unit().split('/')[0] +def principal_unit(): + """Returns the principal unit of this unit, otherwise None""" + # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT + principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None) + # If it's empty, then this unit is the principal + if principal_unit == '': + return os.environ['JUJU_UNIT_NAME'] + elif principal_unit is not None: + return principal_unit + # For Juju 2.1 and below, let's try work out the principle unit by + # the various charms' metadata.yaml. + for reltype in relation_types(): + for rid in relation_ids(reltype): + for unit in related_units(rid): + md = _metadata_unit(unit) + subordinate = md.pop('subordinate', None) + if not subordinate: + return unit + return None + + @cached def remote_service_name(relid=None): """The remote service name for a given relation-id (or the current relation)""" @@ -478,6 +499,21 @@ def metadata(): return yaml.safe_load(md) +def _metadata_unit(unit): + """Given the name of a unit (e.g. apache2/0), get the unit charm's + metadata.yaml. Very similar to metadata() but allows us to inspect + other units. Unit needs to be co-located, such as a subordinate or + principal/primary. + + :returns: metadata.yaml as a python object. + + """ + basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) + unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) + with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md: + return yaml.safe_load(md) + + @cached def relation_types(): """Get a list of relation types supported by this charm""" @@ -753,6 +789,9 @@ class Hooks(object): def charm_dir(): """Return the root directory of the current charm""" + d = os.environ.get('JUJU_CHARM_DIR') + if d is not None: + return d return os.environ.get('CHARM_DIR') diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py index 88e80a49..b0043cbe 100644 --- a/tests/charmhelpers/core/host.py +++ b/tests/charmhelpers/core/host.py @@ -191,6 +191,7 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(): + service('disable', service_name) service('mask', service_name) elif os.path.exists(upstart_file): override_path = os.path.join( @@ -225,6 +226,7 @@ def service_resume(service_name, init_dir="/etc/init", sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(): service('unmask', service_name) + service('enable', service_name) elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name))