From fe4a370121654da8b275e28e420b676ee08a304c Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 24 Jul 2017 14:45:17 -0700 Subject: [PATCH] Update tests to use keystoneauth1 With keystoneauth1 sessions the service catalog search function, url_for takes argument interface, no longer named endpoint_type. This change updates tests to use this argument name. Change-Id: I11d2862d6cbc7282b84fa1bddb64ef44bc2b788d --- hooks/charmhelpers/contrib/network/ip.py | 6 +- .../contrib/openstack/amulet/utils.py | 101 ++++++++---- .../charmhelpers/contrib/openstack/context.py | 20 +++ .../contrib/openstack/keystone.py | 2 +- .../contrib/openstack/templates/ceph.conf | 5 +- .../contrib/openstack/templates/haproxy.cfg | 4 +- hooks/charmhelpers/contrib/openstack/utils.py | 145 +++++++++++++++++- .../contrib/storage/linux/bcache.py | 74 +++++++++ .../contrib/storage/linux/ceph.py | 44 +++++- hooks/charmhelpers/fetch/snap.py | 22 ++- hooks/charmhelpers/fetch/ubuntu.py | 9 +- tests/basic_deployment.py | 2 +- .../contrib/openstack/amulet/utils.py | 101 ++++++++---- 13 files changed, 442 insertions(+), 93 deletions(-) create mode 100644 hooks/charmhelpers/contrib/storage/linux/bcache.py diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index e0796c1..d7e6deb 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -243,11 +243,13 @@ def is_ipv6_disabled(): try: result = subprocess.check_output( ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], - stderr=subprocess.STDOUT).decode('UTF-8') - return "net.ipv6.conf.all.disable_ipv6 = 1" in result + stderr=subprocess.STDOUT, + universal_newlines=True) except subprocess.CalledProcessError: return True + return "net.ipv6.conf.all.disable_ipv6 = 1" in result + def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index bcef4cd..c8edbf6 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -25,9 +25,12 @@ import urlparse import cinderclient.v1.client as cinder_client import glanceclient.v1.client as glance_client import heatclient.v1.client as heat_client -import keystoneclient.v2_0 as keystone_client -from keystoneclient.auth.identity import v3 as keystone_id_v3 -from keystoneclient import session as keystone_session +from keystoneclient.v2_0 import client as keystone_client +from keystoneauth1.identity import ( + v3, + v2, +) +from keystoneauth1 import session as keystone_session from keystoneclient.v3 import client as keystone_client_v3 from novaclient import exceptions @@ -368,12 +371,20 @@ class OpenStackAmuletUtils(AmuletUtils): port) if not api_version or api_version == 2: ep = base_ep + "/v2.0" - return keystone_client.Client(username=username, password=password, - tenant_name=project_name, - auth_url=ep) + auth = v2.Password( + username=username, + password=password, + tenant_name=project_name, + auth_url=ep + ) + sess = keystone_session.Session(auth=auth) + client = keystone_client.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client else: ep = base_ep + "/v3" - auth = keystone_id_v3.Password( + auth = v3.Password( user_domain_name=user_domain_name, username=username, password=password, @@ -382,36 +393,45 @@ class OpenStackAmuletUtils(AmuletUtils): project_name=project_name, auth_url=ep ) - return keystone_client_v3.Client( - session=keystone_session.Session(auth=auth) - ) + sess = keystone_session.Session(auth=auth) + client = keystone_client_v3.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client def authenticate_keystone_admin(self, keystone_sentry, user, password, tenant=None, api_version=None, - keystone_ip=None): + keystone_ip=None, user_domain_name=None, + project_domain_name=None, + project_name=None): """Authenticates admin user with the keystone admin endpoint.""" self.log.debug('Authenticating keystone admin...') if not keystone_ip: keystone_ip = keystone_sentry.info['public-address'] - user_domain_name = None - domain_name = None - if api_version == 3: + # To support backward compatibility usage of this function + if not project_name: + project_name = tenant + if api_version == 3 and not user_domain_name: user_domain_name = 'admin_domain' - domain_name = user_domain_name + if api_version == 3 and not project_domain_name: + project_domain_name = 'admin_domain' + if api_version == 3 and not project_name: + project_name = 'admin' - return self.authenticate_keystone(keystone_ip, user, password, - project_name=tenant, - api_version=api_version, - user_domain_name=user_domain_name, - domain_name=domain_name, - admin_port=True) + return self.authenticate_keystone( + keystone_ip, user, password, + api_version=api_version, + user_domain_name=user_domain_name, + project_domain_name=project_domain_name, + project_name=project_name, + admin_port=True) def authenticate_keystone_user(self, keystone, user, password, tenant): """Authenticates a regular user with the keystone public endpoint.""" self.log.debug('Authenticating keystone user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') keystone_ip = urlparse.urlparse(ep).hostname return self.authenticate_keystone(keystone_ip, user, password, @@ -421,22 +441,32 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates admin user with glance.""" self.log.debug('Authenticating glance admin...') ep = keystone.service_catalog.url_for(service_type='image', - endpoint_type='adminURL') - return glance_client.Client(ep, token=keystone.auth_token) + interface='adminURL') + if keystone.session: + return glance_client.Client(ep, session=keystone.session) + else: + return glance_client.Client(ep, token=keystone.auth_token) def authenticate_heat_admin(self, keystone): """Authenticates the admin user with heat.""" self.log.debug('Authenticating heat admin...') ep = keystone.service_catalog.url_for(service_type='orchestration', - endpoint_type='publicURL') - return heat_client.Client(endpoint=ep, token=keystone.auth_token) + interface='publicURL') + if keystone.session: + return heat_client.Client(endpoint=ep, session=keystone.session) + else: + return heat_client.Client(endpoint=ep, token=keystone.auth_token) def authenticate_nova_user(self, keystone, user, password, tenant): """Authenticates a regular user with nova-api.""" self.log.debug('Authenticating nova user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - if novaclient.__version__[0] >= "7": + interface='publicURL') + if keystone.session: + return nova_client.Client(NOVA_CLIENT_VERSION, + session=keystone.session, + auth_url=ep) + elif novaclient.__version__[0] >= "7": return nova_client.Client(NOVA_CLIENT_VERSION, username=user, password=password, project_name=tenant, auth_url=ep) @@ -449,12 +479,15 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates a regular user with swift api.""" self.log.debug('Authenticating swift user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - return swiftclient.Connection(authurl=ep, - user=user, - key=password, - tenant_name=tenant, - auth_version='2.0') + interface='publicURL') + if keystone.session: + return swiftclient.Connection(session=keystone.session) + else: + return swiftclient.Connection(authurl=ep, + user=user, + key=password, + tenant_name=tenant, + auth_version='2.0') def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto", ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True): diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ea93159..74ceb62 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -97,6 +97,7 @@ from charmhelpers.contrib.openstack.utils import ( git_determine_usr_bin, git_determine_python_path, enable_memcache, + snap_install_requested, ) from charmhelpers.core.unitdata import kv @@ -244,6 +245,11 @@ class SharedDBContext(OSContextGenerator): 'database_password': rdata.get(password_setting), 'database_type': 'mysql' } + # Note(coreycb): We can drop mysql+pymysql if we want when the + # following review lands, though it seems mysql+pymysql would + # be preferred. https://review.openstack.org/#/c/462190/ + if snap_install_requested(): + ctxt['database_type'] = 'mysql+pymysql' if self.context_complete(ctxt): db_ssl(rdata, ctxt, self.ssl_dir) return ctxt @@ -510,6 +516,8 @@ class CephContext(OSContextGenerator): ctxt['auth'] = relation_get('auth', rid=rid, unit=unit) if not ctxt.get('key'): ctxt['key'] = relation_get('key', rid=rid, unit=unit) + if not ctxt.get('rbd_features'): + ctxt['rbd_features'] = relation_get('rbd-features', rid=rid, unit=unit) ceph_addrs = relation_get('ceph-public-address', rid=rid, unit=unit) @@ -1397,6 +1405,18 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'dns-domain', 'default': None, }, + 'polling_interval': { + 'rel_key': 'polling-interval', + 'default': 2, + }, + 'rpc_response_timeout': { + 'rel_key': 'rpc-response-timeout', + 'default': 60, + }, + 'report_interval': { + 'rel_key': 'report-interval', + 'default': 30, + }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): diff --git a/hooks/charmhelpers/contrib/openstack/keystone.py b/hooks/charmhelpers/contrib/openstack/keystone.py index a15a03f..d7e02cc 100644 --- a/hooks/charmhelpers/contrib/openstack/keystone.py +++ b/hooks/charmhelpers/contrib/openstack/keystone.py @@ -29,7 +29,7 @@ def get_api_suffix(api_version): @returns the api suffix formatted according to the given api version """ - return 'v2.0' if api_version in (2, "2.0") else 'v3' + return 'v2.0' if api_version in (2, "2", "2.0") else 'v3' def format_endpoint(schema, addr, port, api_version): diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 33ceee2..ed5c4f1 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -1,6 +1,6 @@ ############################################################################### # [ WARNING ] -# cinder configuration file maintained by Juju +# ceph configuration file maintained by Juju # local changes may be overwritten. ############################################################################### [global] @@ -12,6 +12,9 @@ mon host = {{ mon_hosts }} log to syslog = {{ use_syslog }} err to syslog = {{ use_syslog }} clog to syslog = {{ use_syslog }} +{% if rbd_features %} +rbd default features = {{ rbd_features }} +{% endif %} [client] {% if rbd_client_cache_settings -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 54fba39..edae7a0 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -1,6 +1,6 @@ global - log {{ local_host }} local0 - log {{ local_host }} local1 notice + log /var/lib/haproxy/dev/log local0 + log /var/lib/haproxy/dev/log local1 notice maxconn 20000 user haproxy group haproxy diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 1eaab95..d15a631 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import ( status_set, hook_name, application_version_set, + cached, ) from charmhelpers.core.strutils import BasicStringComparator @@ -90,6 +91,13 @@ from charmhelpers.fetch import ( GPGKeyError, get_upstream_version ) + +from charmhelpers.fetch.snap import ( + snap_install, + snap_refresh, + SNAP_CHANNELS, +) + from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device from charmhelpers.contrib.openstack.exceptions import OSContextError @@ -327,8 +335,10 @@ def get_os_codename_install_source(src): return ca_rel # Best guess match based on deb string provided - if src.startswith('deb') or src.startswith('ppa'): - for k, v in six.iteritems(OPENSTACK_CODENAMES): + if (src.startswith('deb') or + src.startswith('ppa') or + src.startswith('snap')): + for v in OPENSTACK_CODENAMES.values(): if v in src: return v @@ -397,6 +407,19 @@ def get_swift_codename(version): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + + if snap_install_requested(): + cmd = ['snap', 'list', package] + try: + out = subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + return None + lines = out.split('\n') + for line in lines: + if package in line: + # Second item in list is Version + return line.split()[1] + import apt_pkg as apt cache = apt_cache() @@ -613,6 +636,9 @@ def openstack_upgrade_available(package): import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) + if not cur_vers: + # The package has not been installed yet do not attempt upgrade + return False if "swift" in package: codename = get_os_codename_install_source(src) avail_vers = get_os_version_codename_swift(codename) @@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False, return wrap +def ordered(orderme): + """Converts the provided dictionary into a collections.OrderedDict. + + The items in the returned OrderedDict will be inserted based on the + natural sort order of the keys. Nested dictionaries will also be sorted + in order to ensure fully predictable ordering. + + :param orderme: the dict to order + :return: collections.OrderedDict + :raises: ValueError: if `orderme` isn't a dict instance. + """ + if not isinstance(orderme, dict): + raise ValueError('argument must be a dict type') + + result = OrderedDict() + for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]): + if isinstance(v, dict): + result[k] = ordered(v) + else: + result[k] = v + + return result + + def config_flags_parser(config_flags): """Parses config flags string into dict. @@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags): example, a string in the format of 'key1=value1, key2=value2' will return a dict of: - {'key1': 'value1', - 'key2': 'value2'}. + {'key1': 'value1', 'key2': 'value2'}. 2. A string in the above format, but supporting a comma-delimited list of values for the same key. For example, a string in the format of 'key1=value1, key2=value3,value4,value5' will return a dict of: - {'key1', 'value1', - 'key2', 'value2,value3,value4'} + {'key1': 'value1', 'key2': 'value2,value3,value4'} 3. A string containing a colon character (:) prior to an equal character (=) will be treated as yaml and parsed as such. This can be @@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags): equals = config_flags.find('=') if colon > 0: if colon < equals or equals < 0: - return yaml.safe_load(config_flags) + return ordered(yaml.safe_load(config_flags)) if config_flags.find('==') >= 0: juju_log("config_flags is not in expected format (key=value)", @@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags): # split on '='. split = config_flags.strip(' =').split('=') limit = len(split) - flags = {} + flags = OrderedDict() for i in range(0, limit - 1): current = split[i] next = split[i + 1] @@ -1982,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None): if enable_memcache(source=source, release=release): packages.extend(['memcached', 'python-memcache']) return packages + + +def update_json_file(filename, items): + """Updates the json `filename` with a given dict. + :param filename: json filename (i.e.: /etc/glance/policy.json) + :param items: dict of items to update + """ + with open(filename) as fd: + policy = json.load(fd) + policy.update(items) + with open(filename, "w") as fd: + fd.write(json.dumps(policy, indent=4)) + + +@cached +def snap_install_requested(): + """ Determine if installing from snaps + + If openstack-origin is of the form snap:channel-series-release + and channel is in SNAPS_CHANNELS return True. + """ + origin = config('openstack-origin') + if not origin.startswith('snap:'): + return False + + _src = origin[5:] + channel, series, release = _src.split('-') + if channel.lower() in SNAP_CHANNELS: + return True + return False + + +def get_snaps_install_info_from_origin(snaps, src, mode='classic'): + """Generate a dictionary of snap install information from origin + + @param snaps: List of snaps + @param src: String of openstack-origin or source of the form + snap:channel-series-track + @param mode: String classic, devmode or jailmode + @returns: Dictionary of snaps with channels and modes + """ + + if not src.startswith('snap:'): + juju_log("Snap source is not a snap origin", 'WARN') + return {} + + _src = src[5:] + _channel, _series, _release = _src.split('-') + channel = '--channel={}/{}'.format(_release, _channel) + + return {snap: {'channel': channel, 'mode': mode} + for snap in snaps} + + +def install_os_snaps(snaps, refresh=False): + """Install OpenStack snaps from channel and with mode + + @param snaps: Dictionary of snaps with channels and modes of the form: + {'snap_name': {'channel': 'snap_channel', + 'mode': 'snap_mode'}} + Where channel a snapstore channel and mode is --classic, --devmode or + --jailmode. + @param post_snap_install: Callback function to run after snaps have been + installed + """ + + def _ensure_flag(flag): + if flag.startswith('--'): + return flag + return '--{}'.format(flag) + + if refresh: + for snap in snaps.keys(): + snap_refresh(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) + else: + for snap in snaps.keys(): + snap_install(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) diff --git a/hooks/charmhelpers/contrib/storage/linux/bcache.py b/hooks/charmhelpers/contrib/storage/linux/bcache.py new file mode 100644 index 0000000..605991e --- /dev/null +++ b/hooks/charmhelpers/contrib/storage/linux/bcache.py @@ -0,0 +1,74 @@ +# Copyright 2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import json + +from charmhelpers.core.hookenv import log + +stats_intervals = ['stats_day', 'stats_five_minute', + 'stats_hour', 'stats_total'] + +SYSFS = '/sys' + + +class Bcache(object): + """Bcache behaviour + """ + + def __init__(self, cachepath): + self.cachepath = cachepath + + @classmethod + def fromdevice(cls, devname): + return cls('{}/block/{}/bcache'.format(SYSFS, devname)) + + def __str__(self): + return self.cachepath + + def get_stats(self, interval): + """Get cache stats + """ + intervaldir = 'stats_{}'.format(interval) + path = "{}/{}".format(self.cachepath, intervaldir) + out = dict() + for elem in os.listdir(path): + out[elem] = open('{}/{}'.format(path, elem)).read().strip() + return out + + +def get_bcache_fs(): + """Return all cache sets + """ + cachesetroot = "{}/fs/bcache".format(SYSFS) + try: + dirs = os.listdir(cachesetroot) + except OSError: + log("No bcache fs found") + return [] + cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')]) + return cacheset + + +def get_stats_action(cachespec, interval): + """Action for getting bcache statistics for a given cachespec. + Cachespec can either be a device name, eg. 'sdb', which will retrieve + cache stats for the given device, or 'global', which will retrieve stats + for all cachesets + """ + if cachespec == 'global': + caches = get_bcache_fs() + else: + caches = [Bcache.fromdevice(cachespec)] + res = dict((c.cachepath, c.get_stats(interval)) for c in caches) + return json.dumps(res, indent=4, separators=(',', ': ')) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 9417d68..e5a01b1 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -63,6 +63,7 @@ from charmhelpers.core.host import ( from charmhelpers.fetch import ( apt_install, ) +from charmhelpers.core.unitdata import kv from charmhelpers.core.kernel import modprobe from charmhelpers.contrib.openstack.utils import config_flags_parser @@ -1314,6 +1315,47 @@ def send_request_if_needed(request, relation='ceph'): relation_set(relation_id=rid, broker_req=request.request) +def is_broker_action_done(action, rid=None, unit=None): + """Check whether broker action has completed yet. + + @param action: name of action to be performed + @returns True if action complete otherwise False + """ + rdata = relation_get(rid, unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + if not broker_rsp: + return False + + rsp = CephBrokerRsp(broker_rsp) + unit_name = local_unit().partition('/')[2] + key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) + kvstore = kv() + val = kvstore.get(key=key) + if val and val == rsp.request_id: + return True + + return False + + +def mark_broker_action_done(action, rid=None, unit=None): + """Mark action as having been completed. + + @param action: name of action to be performed + @returns None + """ + rdata = relation_get(rid, unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + if not broker_rsp: + return + + rsp = CephBrokerRsp(broker_rsp) + unit_name = local_unit().partition('/')[2] + key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) + kvstore = kv() + kvstore.set(key=key, value=rsp.request_id) + kvstore.flush() + + class CephConfContext(object): """Ceph config (ceph.conf) context. @@ -1330,7 +1372,7 @@ class CephConfContext(object): return {} conf = config_flags_parser(conf) - if type(conf) != dict: + if not isinstance(conf, dict): log("Provided config-flags is not a dictionary - ignoring", level=WARNING) return {} diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py index 23c707b..112a54c 100644 --- a/hooks/charmhelpers/fetch/snap.py +++ b/hooks/charmhelpers/fetch/snap.py @@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer: https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html """ import subprocess -from os import environ +import os from time import sleep from charmhelpers.core.hookenv import log __author__ = 'Joseph Borg ' -SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +# The return code for "couldn't acquire lock" in Snap +# (hopefully this will be improved). +SNAP_NO_LOCK = 1 SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +SNAP_CHANNELS = [ + 'edge', + 'beta', + 'candidate', + 'stable', +] class CouldNotAcquireLockException(Exception): @@ -47,13 +55,17 @@ def _snap_exec(commands): while return_code is None or return_code == SNAP_NO_LOCK: try: - return_code = subprocess.check_call(['snap'] + commands, env=environ) + return_code = subprocess.check_call(['snap'] + commands, + env=os.environ) except subprocess.CalledProcessError as e: retry_count += + 1 if retry_count > SNAP_NO_LOCK_RETRY_COUNT: - raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + raise CouldNotAcquireLockException( + 'Could not aquire lock after {} attempts' + .format(SNAP_NO_LOCK_RETRY_COUNT)) return_code = e.returncode - log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + log('Snap failed to acquire lock, trying again in {} seconds.' + .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN')) sleep(SNAP_NO_LOCK_RETRY_DELAY) return return_code diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 57b5fb6..545348f 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -139,7 +139,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/ocata': 'xenial-updates/ocata', 'ocata/proposed': 'xenial-proposed/ocata', 'xenial-ocata/proposed': 'xenial-proposed/ocata', - 'xenial-ocata/newton': 'xenial-proposed/ocata', + 'xenial-proposed/ocata': 'xenial-proposed/ocata', # Pike 'pike': 'xenial-updates/pike', 'xenial-pike': 'xenial-updates/pike', @@ -147,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/pike': 'xenial-updates/pike', 'pike/proposed': 'xenial-proposed/pike', 'xenial-pike/proposed': 'xenial-proposed/pike', - 'xenial-pike/newton': 'xenial-proposed/pike', + 'xenial-proposed/pike': 'xenial-proposed/pike', # Queens 'queens': 'xenial-updates/queens', 'xenial-queens': 'xenial-updates/queens', @@ -155,13 +155,13 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/queens': 'xenial-updates/queens', 'queens/proposed': 'xenial-proposed/queens', 'xenial-queens/proposed': 'xenial-proposed/queens', - 'xenial-queens/newton': 'xenial-proposed/queens', + 'xenial-proposed/queens': 'xenial-proposed/queens', } APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. +CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -364,6 +364,7 @@ def add_source(source, key=None, fail_invalid=False): (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), (r"^cloud:(.*)$", _add_cloud_pocket), + (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), ]) if source is None: source = '' diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index addff82..97e9ef6 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -132,7 +132,7 @@ class CeiloAgentBasicDeployment(OpenStackAmuletDeployment): # Authenticate admin with ceilometer endpoint ep = self.keystone.service_catalog.url_for(service_type='metering', - endpoint_type='publicURL') + interface='publicURL') os_token = self.keystone.auth_token self.log.debug('Instantiating ceilometer client...') self.ceil = ceilo_client.Client(endpoint=ep, token=os_token) diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index bcef4cd..c8edbf6 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -25,9 +25,12 @@ import urlparse import cinderclient.v1.client as cinder_client import glanceclient.v1.client as glance_client import heatclient.v1.client as heat_client -import keystoneclient.v2_0 as keystone_client -from keystoneclient.auth.identity import v3 as keystone_id_v3 -from keystoneclient import session as keystone_session +from keystoneclient.v2_0 import client as keystone_client +from keystoneauth1.identity import ( + v3, + v2, +) +from keystoneauth1 import session as keystone_session from keystoneclient.v3 import client as keystone_client_v3 from novaclient import exceptions @@ -368,12 +371,20 @@ class OpenStackAmuletUtils(AmuletUtils): port) if not api_version or api_version == 2: ep = base_ep + "/v2.0" - return keystone_client.Client(username=username, password=password, - tenant_name=project_name, - auth_url=ep) + auth = v2.Password( + username=username, + password=password, + tenant_name=project_name, + auth_url=ep + ) + sess = keystone_session.Session(auth=auth) + client = keystone_client.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client else: ep = base_ep + "/v3" - auth = keystone_id_v3.Password( + auth = v3.Password( user_domain_name=user_domain_name, username=username, password=password, @@ -382,36 +393,45 @@ class OpenStackAmuletUtils(AmuletUtils): project_name=project_name, auth_url=ep ) - return keystone_client_v3.Client( - session=keystone_session.Session(auth=auth) - ) + sess = keystone_session.Session(auth=auth) + client = keystone_client_v3.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client def authenticate_keystone_admin(self, keystone_sentry, user, password, tenant=None, api_version=None, - keystone_ip=None): + keystone_ip=None, user_domain_name=None, + project_domain_name=None, + project_name=None): """Authenticates admin user with the keystone admin endpoint.""" self.log.debug('Authenticating keystone admin...') if not keystone_ip: keystone_ip = keystone_sentry.info['public-address'] - user_domain_name = None - domain_name = None - if api_version == 3: + # To support backward compatibility usage of this function + if not project_name: + project_name = tenant + if api_version == 3 and not user_domain_name: user_domain_name = 'admin_domain' - domain_name = user_domain_name + if api_version == 3 and not project_domain_name: + project_domain_name = 'admin_domain' + if api_version == 3 and not project_name: + project_name = 'admin' - return self.authenticate_keystone(keystone_ip, user, password, - project_name=tenant, - api_version=api_version, - user_domain_name=user_domain_name, - domain_name=domain_name, - admin_port=True) + return self.authenticate_keystone( + keystone_ip, user, password, + api_version=api_version, + user_domain_name=user_domain_name, + project_domain_name=project_domain_name, + project_name=project_name, + admin_port=True) def authenticate_keystone_user(self, keystone, user, password, tenant): """Authenticates a regular user with the keystone public endpoint.""" self.log.debug('Authenticating keystone user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') keystone_ip = urlparse.urlparse(ep).hostname return self.authenticate_keystone(keystone_ip, user, password, @@ -421,22 +441,32 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates admin user with glance.""" self.log.debug('Authenticating glance admin...') ep = keystone.service_catalog.url_for(service_type='image', - endpoint_type='adminURL') - return glance_client.Client(ep, token=keystone.auth_token) + interface='adminURL') + if keystone.session: + return glance_client.Client(ep, session=keystone.session) + else: + return glance_client.Client(ep, token=keystone.auth_token) def authenticate_heat_admin(self, keystone): """Authenticates the admin user with heat.""" self.log.debug('Authenticating heat admin...') ep = keystone.service_catalog.url_for(service_type='orchestration', - endpoint_type='publicURL') - return heat_client.Client(endpoint=ep, token=keystone.auth_token) + interface='publicURL') + if keystone.session: + return heat_client.Client(endpoint=ep, session=keystone.session) + else: + return heat_client.Client(endpoint=ep, token=keystone.auth_token) def authenticate_nova_user(self, keystone, user, password, tenant): """Authenticates a regular user with nova-api.""" self.log.debug('Authenticating nova user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - if novaclient.__version__[0] >= "7": + interface='publicURL') + if keystone.session: + return nova_client.Client(NOVA_CLIENT_VERSION, + session=keystone.session, + auth_url=ep) + elif novaclient.__version__[0] >= "7": return nova_client.Client(NOVA_CLIENT_VERSION, username=user, password=password, project_name=tenant, auth_url=ep) @@ -449,12 +479,15 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates a regular user with swift api.""" self.log.debug('Authenticating swift user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - return swiftclient.Connection(authurl=ep, - user=user, - key=password, - tenant_name=tenant, - auth_version='2.0') + interface='publicURL') + if keystone.session: + return swiftclient.Connection(session=keystone.session) + else: + return swiftclient.Connection(authurl=ep, + user=user, + key=password, + tenant_name=tenant, + auth_version='2.0') def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto", ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):