From 649ea8d04095f14b114f6725208e3cec892a025a Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 24 Jul 2017 14:46:04 -0700 Subject: [PATCH] Update tests to use keystoneauth1 With keystoneauth1 sessions the service catalog search function, url_for takes argument interface, no longer named endpoint_type. This change updates tests to use this argument name. Change-Id: I1c2100d47de440fe8a53309b130b0634288af84c --- hooks/charmhelpers/__init__.py | 61 ++++ hooks/charmhelpers/contrib/network/ip.py | 6 +- .../contrib/openstack/amulet/utils.py | 101 +++-- .../charmhelpers/contrib/openstack/context.py | 20 + .../contrib/openstack/keystone.py | 2 +- .../contrib/openstack/templates/ceph.conf | 5 +- .../contrib/openstack/templates/haproxy.cfg | 4 +- hooks/charmhelpers/contrib/openstack/utils.py | 345 ++++++++++-------- .../contrib/storage/linux/bcache.py | 74 ++++ .../contrib/storage/linux/ceph.py | 44 ++- hooks/charmhelpers/fetch/__init__.py | 26 +- hooks/charmhelpers/fetch/centos.py | 2 +- hooks/charmhelpers/fetch/snap.py | 22 +- hooks/charmhelpers/fetch/ubuntu.py | 309 +++++++++++++--- tests/basic_deployment.py | 8 +- tests/charmhelpers/__init__.py | 61 ++++ tests/charmhelpers/contrib/network/ip.py | 6 +- .../contrib/openstack/amulet/utils.py | 101 +++-- tests/charmhelpers/contrib/openstack/utils.py | 345 ++++++++++-------- .../contrib/storage/linux/bcache.py | 74 ++++ .../contrib/storage/linux/ceph.py | 44 ++- tests/charmhelpers/fetch/__init__.py | 26 +- tests/charmhelpers/fetch/centos.py | 2 +- tests/charmhelpers/fetch/snap.py | 22 +- tests/charmhelpers/fetch/ubuntu.py | 309 +++++++++++++--- 25 files changed, 1501 insertions(+), 518 deletions(-) create mode 100644 hooks/charmhelpers/contrib/storage/linux/bcache.py create mode 100644 tests/charmhelpers/contrib/storage/linux/bcache.py diff --git a/hooks/charmhelpers/__init__.py b/hooks/charmhelpers/__init__.py index 48867880..e7aa4715 100644 --- a/hooks/charmhelpers/__init__.py +++ b/hooks/charmhelpers/__init__.py @@ -14,6 +14,11 @@ # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. +from __future__ import print_function +from __future__ import absolute_import + +import functools +import inspect import subprocess import sys @@ -34,3 +39,59 @@ except ImportError: else: subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) import yaml # flake8: noqa + + +# Holds a list of mapping of mangled function names that have been deprecated +# using the @deprecate decorator below. This is so that the warning is only +# printed once for each usage of the function. +__deprecated_functions = {} + + +def deprecate(warning, date=None, log=None): + """Add a deprecation warning the first time the function is used. + The date, which is a string in semi-ISO8660 format indicate the year-month + that the function is officially going to be removed. + + usage: + + @deprecate('use core/fetch/add_source() instead', '2017-04') + def contributed_add_source_thing(...): + ... + + And it then prints to the log ONCE that the function is deprecated. + The reason for passing the logging function (log) is so that hookenv.log + can be used for a charm if needed. + + :param warning: String to indicat where it has moved ot. + :param date: optional sting, in YYYY-MM format to indicate when the + function will definitely (probably) be removed. + :param log: The log function to call to log. If not, logs to stdout + """ + def wrap(f): + + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + try: + module = inspect.getmodule(f) + file = inspect.getsourcefile(f) + lines = inspect.getsourcelines(f) + f_name = "{}-{}-{}..{}-{}".format( + module.__name__, file, lines[0], lines[-1], f.__name__) + except (IOError, TypeError): + # assume it was local, so just use the name of the function + f_name = f.__name__ + if f_name not in __deprecated_functions: + __deprecated_functions[f_name] = True + s = "DEPRECATION WARNING: Function {} is being removed".format( + f.__name__) + if date: + s = "{} on/around {}".format(s, date) + if warning: + s = "{} : {}".format(s, warning) + if log: + log(s) + else: + print(s) + return f(*args, **kwargs) + return wrapped_f + return wrap diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index fc3f5e3e..d7e6debf 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -243,11 +243,13 @@ def is_ipv6_disabled(): try: result = subprocess.check_output( ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], - stderr=subprocess.STDOUT) - return "net.ipv6.conf.all.disable_ipv6 = 1" in result + stderr=subprocess.STDOUT, + universal_newlines=True) except subprocess.CalledProcessError: return True + return "net.ipv6.conf.all.disable_ipv6 = 1" in result + def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index bcef4cd0..c8edbf65 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -25,9 +25,12 @@ import urlparse import cinderclient.v1.client as cinder_client import glanceclient.v1.client as glance_client import heatclient.v1.client as heat_client -import keystoneclient.v2_0 as keystone_client -from keystoneclient.auth.identity import v3 as keystone_id_v3 -from keystoneclient import session as keystone_session +from keystoneclient.v2_0 import client as keystone_client +from keystoneauth1.identity import ( + v3, + v2, +) +from keystoneauth1 import session as keystone_session from keystoneclient.v3 import client as keystone_client_v3 from novaclient import exceptions @@ -368,12 +371,20 @@ class OpenStackAmuletUtils(AmuletUtils): port) if not api_version or api_version == 2: ep = base_ep + "/v2.0" - return keystone_client.Client(username=username, password=password, - tenant_name=project_name, - auth_url=ep) + auth = v2.Password( + username=username, + password=password, + tenant_name=project_name, + auth_url=ep + ) + sess = keystone_session.Session(auth=auth) + client = keystone_client.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client else: ep = base_ep + "/v3" - auth = keystone_id_v3.Password( + auth = v3.Password( user_domain_name=user_domain_name, username=username, password=password, @@ -382,36 +393,45 @@ class OpenStackAmuletUtils(AmuletUtils): project_name=project_name, auth_url=ep ) - return keystone_client_v3.Client( - session=keystone_session.Session(auth=auth) - ) + sess = keystone_session.Session(auth=auth) + client = keystone_client_v3.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client def authenticate_keystone_admin(self, keystone_sentry, user, password, tenant=None, api_version=None, - keystone_ip=None): + keystone_ip=None, user_domain_name=None, + project_domain_name=None, + project_name=None): """Authenticates admin user with the keystone admin endpoint.""" self.log.debug('Authenticating keystone admin...') if not keystone_ip: keystone_ip = keystone_sentry.info['public-address'] - user_domain_name = None - domain_name = None - if api_version == 3: + # To support backward compatibility usage of this function + if not project_name: + project_name = tenant + if api_version == 3 and not user_domain_name: user_domain_name = 'admin_domain' - domain_name = user_domain_name + if api_version == 3 and not project_domain_name: + project_domain_name = 'admin_domain' + if api_version == 3 and not project_name: + project_name = 'admin' - return self.authenticate_keystone(keystone_ip, user, password, - project_name=tenant, - api_version=api_version, - user_domain_name=user_domain_name, - domain_name=domain_name, - admin_port=True) + return self.authenticate_keystone( + keystone_ip, user, password, + api_version=api_version, + user_domain_name=user_domain_name, + project_domain_name=project_domain_name, + project_name=project_name, + admin_port=True) def authenticate_keystone_user(self, keystone, user, password, tenant): """Authenticates a regular user with the keystone public endpoint.""" self.log.debug('Authenticating keystone user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') keystone_ip = urlparse.urlparse(ep).hostname return self.authenticate_keystone(keystone_ip, user, password, @@ -421,22 +441,32 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates admin user with glance.""" self.log.debug('Authenticating glance admin...') ep = keystone.service_catalog.url_for(service_type='image', - endpoint_type='adminURL') - return glance_client.Client(ep, token=keystone.auth_token) + interface='adminURL') + if keystone.session: + return glance_client.Client(ep, session=keystone.session) + else: + return glance_client.Client(ep, token=keystone.auth_token) def authenticate_heat_admin(self, keystone): """Authenticates the admin user with heat.""" self.log.debug('Authenticating heat admin...') ep = keystone.service_catalog.url_for(service_type='orchestration', - endpoint_type='publicURL') - return heat_client.Client(endpoint=ep, token=keystone.auth_token) + interface='publicURL') + if keystone.session: + return heat_client.Client(endpoint=ep, session=keystone.session) + else: + return heat_client.Client(endpoint=ep, token=keystone.auth_token) def authenticate_nova_user(self, keystone, user, password, tenant): """Authenticates a regular user with nova-api.""" self.log.debug('Authenticating nova user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - if novaclient.__version__[0] >= "7": + interface='publicURL') + if keystone.session: + return nova_client.Client(NOVA_CLIENT_VERSION, + session=keystone.session, + auth_url=ep) + elif novaclient.__version__[0] >= "7": return nova_client.Client(NOVA_CLIENT_VERSION, username=user, password=password, project_name=tenant, auth_url=ep) @@ -449,12 +479,15 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates a regular user with swift api.""" self.log.debug('Authenticating swift user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - return swiftclient.Connection(authurl=ep, - user=user, - key=password, - tenant_name=tenant, - auth_version='2.0') + interface='publicURL') + if keystone.session: + return swiftclient.Connection(session=keystone.session) + else: + return swiftclient.Connection(authurl=ep, + user=user, + key=password, + tenant_name=tenant, + auth_version='2.0') def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto", ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True): diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ea93159d..74ceb629 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -97,6 +97,7 @@ from charmhelpers.contrib.openstack.utils import ( git_determine_usr_bin, git_determine_python_path, enable_memcache, + snap_install_requested, ) from charmhelpers.core.unitdata import kv @@ -244,6 +245,11 @@ class SharedDBContext(OSContextGenerator): 'database_password': rdata.get(password_setting), 'database_type': 'mysql' } + # Note(coreycb): We can drop mysql+pymysql if we want when the + # following review lands, though it seems mysql+pymysql would + # be preferred. https://review.openstack.org/#/c/462190/ + if snap_install_requested(): + ctxt['database_type'] = 'mysql+pymysql' if self.context_complete(ctxt): db_ssl(rdata, ctxt, self.ssl_dir) return ctxt @@ -510,6 +516,8 @@ class CephContext(OSContextGenerator): ctxt['auth'] = relation_get('auth', rid=rid, unit=unit) if not ctxt.get('key'): ctxt['key'] = relation_get('key', rid=rid, unit=unit) + if not ctxt.get('rbd_features'): + ctxt['rbd_features'] = relation_get('rbd-features', rid=rid, unit=unit) ceph_addrs = relation_get('ceph-public-address', rid=rid, unit=unit) @@ -1397,6 +1405,18 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'dns-domain', 'default': None, }, + 'polling_interval': { + 'rel_key': 'polling-interval', + 'default': 2, + }, + 'rpc_response_timeout': { + 'rel_key': 'rpc-response-timeout', + 'default': 60, + }, + 'report_interval': { + 'rel_key': 'report-interval', + 'default': 30, + }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): diff --git a/hooks/charmhelpers/contrib/openstack/keystone.py b/hooks/charmhelpers/contrib/openstack/keystone.py index a15a03fa..d7e02ccd 100644 --- a/hooks/charmhelpers/contrib/openstack/keystone.py +++ b/hooks/charmhelpers/contrib/openstack/keystone.py @@ -29,7 +29,7 @@ def get_api_suffix(api_version): @returns the api suffix formatted according to the given api version """ - return 'v2.0' if api_version in (2, "2.0") else 'v3' + return 'v2.0' if api_version in (2, "2", "2.0") else 'v3' def format_endpoint(schema, addr, port, api_version): diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 33ceee25..ed5c4f10 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -1,6 +1,6 @@ ############################################################################### # [ WARNING ] -# cinder configuration file maintained by Juju +# ceph configuration file maintained by Juju # local changes may be overwritten. ############################################################################### [global] @@ -12,6 +12,9 @@ mon host = {{ mon_hosts }} log to syslog = {{ use_syslog }} err to syslog = {{ use_syslog }} clog to syslog = {{ use_syslog }} +{% if rbd_features %} +rbd default features = {{ rbd_features }} +{% endif %} [client] {% if rbd_client_cache_settings -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 54fba39d..edae7a0c 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -1,6 +1,6 @@ global - log {{ local_host }} local0 - log {{ local_host }} local1 notice + log /var/lib/haproxy/dev/log local0 + log /var/lib/haproxy/dev/log local1 notice maxconn 20000 user haproxy group haproxy diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 161c786b..d15a6317 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -26,11 +26,12 @@ import functools import shutil import six -import tempfile import traceback import uuid import yaml +from charmhelpers import deprecate + from charmhelpers.contrib.network import ip from charmhelpers.core import unitdata @@ -41,7 +42,6 @@ from charmhelpers.core.hookenv import ( config, log as juju_log, charm_dir, - DEBUG, INFO, ERROR, related_units, @@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import ( status_set, hook_name, application_version_set, + cached, ) from charmhelpers.core.strutils import BasicStringComparator @@ -82,11 +83,21 @@ from charmhelpers.core.host import ( restart_on_change_helper, ) from charmhelpers.fetch import ( - apt_install, apt_cache, install_remote, + import_key as fetch_import_key, + add_source as fetch_add_source, + SourceConfigError, + GPGKeyError, get_upstream_version ) + +from charmhelpers.fetch.snap import ( + snap_install, + snap_refresh, + SNAP_CHANNELS, +) + from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device from charmhelpers.contrib.openstack.exceptions import OSContextError @@ -324,8 +335,10 @@ def get_os_codename_install_source(src): return ca_rel # Best guess match based on deb string provided - if src.startswith('deb') or src.startswith('ppa'): - for k, v in six.iteritems(OPENSTACK_CODENAMES): + if (src.startswith('deb') or + src.startswith('ppa') or + src.startswith('snap')): + for v in OPENSTACK_CODENAMES.values(): if v in src: return v @@ -394,6 +407,19 @@ def get_swift_codename(version): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + + if snap_install_requested(): + cmd = ['snap', 'list', package] + try: + out = subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + return None + lines = out.split('\n') + for line in lines: + if package in line: + # Second item in list is Version + return line.split()[1] + import apt_pkg as apt cache = apt_cache() @@ -469,13 +495,14 @@ def get_os_version_package(pkg, fatal=True): # error_out(e) -os_rel = None +# Module local cache variable for the os_release. +_os_rel = None def reset_os_release(): '''Unset the cached os_release version''' - global os_rel - os_rel = None + global _os_rel + _os_rel = None def os_release(package, base='essex', reset_cache=False): @@ -489,150 +516,77 @@ def os_release(package, base='essex', reset_cache=False): the installation source, the earliest release supported by the charm should be returned. ''' - global os_rel + global _os_rel if reset_cache: reset_os_release() - if os_rel: - return os_rel - os_rel = (git_os_codename_install_source(config('openstack-origin-git')) or - get_os_codename_package(package, fatal=False) or - get_os_codename_install_source(config('openstack-origin')) or - base) - return os_rel + if _os_rel: + return _os_rel + _os_rel = ( + git_os_codename_install_source(config('openstack-origin-git')) or + get_os_codename_package(package, fatal=False) or + get_os_codename_install_source(config('openstack-origin')) or + base) + return _os_rel +@deprecate("moved to charmhelpers.fetch.import_key()", "2017-07", log=juju_log) def import_key(keyid): - key = keyid.strip() - if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and - key.endswith('-----END PGP PUBLIC KEY BLOCK-----')): - juju_log("PGP key found (looks like ASCII Armor format)", level=DEBUG) - juju_log("Importing ASCII Armor PGP key", level=DEBUG) - with tempfile.NamedTemporaryFile() as keyfile: - with open(keyfile.name, 'w') as fd: - fd.write(key) - fd.write("\n") + """Import a key, either ASCII armored, or a GPG key id. - cmd = ['apt-key', 'add', keyfile.name] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - error_out("Error importing PGP key '%s'" % key) - else: - juju_log("PGP key found (looks like Radix64 format)", level=DEBUG) - juju_log("Importing PGP key from keyserver", level=DEBUG) - cmd = ['apt-key', 'adv', '--keyserver', - 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - error_out("Error importing PGP key '%s'" % key) + @param keyid: the key in ASCII armor format, or a GPG key id. + @raises SystemExit() via sys.exit() on failure. + """ + try: + return fetch_import_key(keyid) + except GPGKeyError as e: + error_out("Could not import key: {}".format(str(e))) -def get_source_and_pgp_key(input): - """Look for a pgp key ID or ascii-armor key in the given input.""" - index = input.strip() - index = input.rfind('|') - if index < 0: - return input, None +def get_source_and_pgp_key(source_and_key): + """Look for a pgp key ID or ascii-armor key in the given input. - key = input[index + 1:].strip('|') - source = input[:index] - return source, key + :param source_and_key: Sting, "source_spec|keyid" where '|keyid' is + optional. + :returns (source_spec, key_id OR None) as a tuple. Returns None for key_id + if there was no '|' in the source_and_key string. + """ + try: + source, key = source_and_key.split('|', 2) + return source, key or None + except ValueError: + return source_and_key, None -def configure_installation_source(rel): - '''Configure apt installation source.''' - if rel == 'distro': - return - elif rel == 'distro-proposed': - ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] - with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: - f.write(DISTRO_PROPOSED % ubuntu_rel) - elif rel[:4] == "ppa:": - src, key = get_source_and_pgp_key(rel) - if key: - import_key(key) +@deprecate("use charmhelpers.fetch.add_source() instead.", + "2017-07", log=juju_log) +def configure_installation_source(source_plus_key): + """Configure an installation source. - subprocess.check_call(["add-apt-repository", "-y", src]) - elif rel[:3] == "deb": - src, key = get_source_and_pgp_key(rel) - if key: - import_key(key) + The functionality is provided by charmhelpers.fetch.add_source() + The difference between the two functions is that add_source() signature + requires the key to be passed directly, whereas this function passes an + optional key by appending '|' to the end of the source specificiation + 'source'. - with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: - f.write(src) - elif rel[:6] == 'cloud:': - ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] - rel = rel.split(':')[1] - u_rel = rel.split('-')[0] - ca_rel = rel.split('-')[1] + Another difference from add_source() is that the function calls sys.exit(1) + if the configuration fails, whereas add_source() raises + SourceConfigurationError(). Another difference, is that add_source() + silently fails (with a juju_log command) if there is no matching source to + configure, whereas this function fails with a sys.exit(1) - if u_rel != ubuntu_rel: - e = 'Cannot install from Cloud Archive pocket %s on this Ubuntu '\ - 'version (%s)' % (ca_rel, ubuntu_rel) - error_out(e) + :param source: String_plus_key -- see above for details. - if 'staging' in ca_rel: - # staging is just a regular PPA. - os_rel = ca_rel.split('/')[0] - ppa = 'ppa:ubuntu-cloud-archive/%s-staging' % os_rel - cmd = 'add-apt-repository -y %s' % ppa - subprocess.check_call(cmd.split(' ')) - return + Note that the behaviour on error is to log the error to the juju log and + then call sys.exit(1). + """ + # extract the key if there is one, denoted by a '|' in the rel + source, key = get_source_and_pgp_key(source_plus_key) - # map charm config options to actual archive pockets. - pockets = { - 'folsom': 'precise-updates/folsom', - 'folsom/updates': 'precise-updates/folsom', - 'folsom/proposed': 'precise-proposed/folsom', - 'grizzly': 'precise-updates/grizzly', - 'grizzly/updates': 'precise-updates/grizzly', - 'grizzly/proposed': 'precise-proposed/grizzly', - 'havana': 'precise-updates/havana', - 'havana/updates': 'precise-updates/havana', - 'havana/proposed': 'precise-proposed/havana', - 'icehouse': 'precise-updates/icehouse', - 'icehouse/updates': 'precise-updates/icehouse', - 'icehouse/proposed': 'precise-proposed/icehouse', - 'juno': 'trusty-updates/juno', - 'juno/updates': 'trusty-updates/juno', - 'juno/proposed': 'trusty-proposed/juno', - 'kilo': 'trusty-updates/kilo', - 'kilo/updates': 'trusty-updates/kilo', - 'kilo/proposed': 'trusty-proposed/kilo', - 'liberty': 'trusty-updates/liberty', - 'liberty/updates': 'trusty-updates/liberty', - 'liberty/proposed': 'trusty-proposed/liberty', - 'mitaka': 'trusty-updates/mitaka', - 'mitaka/updates': 'trusty-updates/mitaka', - 'mitaka/proposed': 'trusty-proposed/mitaka', - 'newton': 'xenial-updates/newton', - 'newton/updates': 'xenial-updates/newton', - 'newton/proposed': 'xenial-proposed/newton', - 'ocata': 'xenial-updates/ocata', - 'ocata/updates': 'xenial-updates/ocata', - 'ocata/proposed': 'xenial-proposed/ocata', - 'pike': 'xenial-updates/pike', - 'pike/updates': 'xenial-updates/pike', - 'pike/proposed': 'xenial-proposed/pike', - 'queens': 'xenial-updates/queens', - 'queens/updates': 'xenial-updates/queens', - 'queens/proposed': 'xenial-proposed/queens', - } - - try: - pocket = pockets[ca_rel] - except KeyError: - e = 'Invalid Cloud Archive release specified: %s' % rel - error_out(e) - - src = "deb %s %s main" % (CLOUD_ARCHIVE_URL, pocket) - apt_install('ubuntu-cloud-keyring', fatal=True) - - with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as f: - f.write(src) - else: - error_out("Invalid openstack-release specified: %s" % rel) + # handle the ordinary sources via add_source + try: + fetch_add_source(source, key, fail_invalid=True) + except SourceConfigError as se: + error_out(str(se)) def config_value_changed(option): @@ -677,12 +631,14 @@ def openstack_upgrade_available(package): :returns: bool: : Returns True if configured installation source offers a newer version of package. - """ import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) + if not cur_vers: + # The package has not been installed yet do not attempt upgrade + return False if "swift" in package: codename = get_os_codename_install_source(src) avail_vers = get_os_version_codename_swift(codename) @@ -1933,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False, return wrap +def ordered(orderme): + """Converts the provided dictionary into a collections.OrderedDict. + + The items in the returned OrderedDict will be inserted based on the + natural sort order of the keys. Nested dictionaries will also be sorted + in order to ensure fully predictable ordering. + + :param orderme: the dict to order + :return: collections.OrderedDict + :raises: ValueError: if `orderme` isn't a dict instance. + """ + if not isinstance(orderme, dict): + raise ValueError('argument must be a dict type') + + result = OrderedDict() + for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]): + if isinstance(v, dict): + result[k] = ordered(v) + else: + result[k] = v + + return result + + def config_flags_parser(config_flags): """Parses config flags string into dict. @@ -1944,15 +1924,13 @@ def config_flags_parser(config_flags): example, a string in the format of 'key1=value1, key2=value2' will return a dict of: - {'key1': 'value1', - 'key2': 'value2'}. + {'key1': 'value1', 'key2': 'value2'}. 2. A string in the above format, but supporting a comma-delimited list of values for the same key. For example, a string in the format of 'key1=value1, key2=value3,value4,value5' will return a dict of: - {'key1', 'value1', - 'key2', 'value2,value3,value4'} + {'key1': 'value1', 'key2': 'value2,value3,value4'} 3. A string containing a colon character (:) prior to an equal character (=) will be treated as yaml and parsed as such. This can be @@ -1972,7 +1950,7 @@ def config_flags_parser(config_flags): equals = config_flags.find('=') if colon > 0: if colon < equals or equals < 0: - return yaml.safe_load(config_flags) + return ordered(yaml.safe_load(config_flags)) if config_flags.find('==') >= 0: juju_log("config_flags is not in expected format (key=value)", @@ -1985,7 +1963,7 @@ def config_flags_parser(config_flags): # split on '='. split = config_flags.strip(' =').split('=') limit = len(split) - flags = {} + flags = OrderedDict() for i in range(0, limit - 1): current = split[i] next = split[i + 1] @@ -2052,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None): if enable_memcache(source=source, release=release): packages.extend(['memcached', 'python-memcache']) return packages + + +def update_json_file(filename, items): + """Updates the json `filename` with a given dict. + :param filename: json filename (i.e.: /etc/glance/policy.json) + :param items: dict of items to update + """ + with open(filename) as fd: + policy = json.load(fd) + policy.update(items) + with open(filename, "w") as fd: + fd.write(json.dumps(policy, indent=4)) + + +@cached +def snap_install_requested(): + """ Determine if installing from snaps + + If openstack-origin is of the form snap:channel-series-release + and channel is in SNAPS_CHANNELS return True. + """ + origin = config('openstack-origin') + if not origin.startswith('snap:'): + return False + + _src = origin[5:] + channel, series, release = _src.split('-') + if channel.lower() in SNAP_CHANNELS: + return True + return False + + +def get_snaps_install_info_from_origin(snaps, src, mode='classic'): + """Generate a dictionary of snap install information from origin + + @param snaps: List of snaps + @param src: String of openstack-origin or source of the form + snap:channel-series-track + @param mode: String classic, devmode or jailmode + @returns: Dictionary of snaps with channels and modes + """ + + if not src.startswith('snap:'): + juju_log("Snap source is not a snap origin", 'WARN') + return {} + + _src = src[5:] + _channel, _series, _release = _src.split('-') + channel = '--channel={}/{}'.format(_release, _channel) + + return {snap: {'channel': channel, 'mode': mode} + for snap in snaps} + + +def install_os_snaps(snaps, refresh=False): + """Install OpenStack snaps from channel and with mode + + @param snaps: Dictionary of snaps with channels and modes of the form: + {'snap_name': {'channel': 'snap_channel', + 'mode': 'snap_mode'}} + Where channel a snapstore channel and mode is --classic, --devmode or + --jailmode. + @param post_snap_install: Callback function to run after snaps have been + installed + """ + + def _ensure_flag(flag): + if flag.startswith('--'): + return flag + return '--{}'.format(flag) + + if refresh: + for snap in snaps.keys(): + snap_refresh(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) + else: + for snap in snaps.keys(): + snap_install(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) diff --git a/hooks/charmhelpers/contrib/storage/linux/bcache.py b/hooks/charmhelpers/contrib/storage/linux/bcache.py new file mode 100644 index 00000000..605991e1 --- /dev/null +++ b/hooks/charmhelpers/contrib/storage/linux/bcache.py @@ -0,0 +1,74 @@ +# Copyright 2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import json + +from charmhelpers.core.hookenv import log + +stats_intervals = ['stats_day', 'stats_five_minute', + 'stats_hour', 'stats_total'] + +SYSFS = '/sys' + + +class Bcache(object): + """Bcache behaviour + """ + + def __init__(self, cachepath): + self.cachepath = cachepath + + @classmethod + def fromdevice(cls, devname): + return cls('{}/block/{}/bcache'.format(SYSFS, devname)) + + def __str__(self): + return self.cachepath + + def get_stats(self, interval): + """Get cache stats + """ + intervaldir = 'stats_{}'.format(interval) + path = "{}/{}".format(self.cachepath, intervaldir) + out = dict() + for elem in os.listdir(path): + out[elem] = open('{}/{}'.format(path, elem)).read().strip() + return out + + +def get_bcache_fs(): + """Return all cache sets + """ + cachesetroot = "{}/fs/bcache".format(SYSFS) + try: + dirs = os.listdir(cachesetroot) + except OSError: + log("No bcache fs found") + return [] + cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')]) + return cacheset + + +def get_stats_action(cachespec, interval): + """Action for getting bcache statistics for a given cachespec. + Cachespec can either be a device name, eg. 'sdb', which will retrieve + cache stats for the given device, or 'global', which will retrieve stats + for all cachesets + """ + if cachespec == 'global': + caches = get_bcache_fs() + else: + caches = [Bcache.fromdevice(cachespec)] + res = dict((c.cachepath, c.get_stats(interval)) for c in caches) + return json.dumps(res, indent=4, separators=(',', ': ')) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 9417d684..e5a01b1b 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -63,6 +63,7 @@ from charmhelpers.core.host import ( from charmhelpers.fetch import ( apt_install, ) +from charmhelpers.core.unitdata import kv from charmhelpers.core.kernel import modprobe from charmhelpers.contrib.openstack.utils import config_flags_parser @@ -1314,6 +1315,47 @@ def send_request_if_needed(request, relation='ceph'): relation_set(relation_id=rid, broker_req=request.request) +def is_broker_action_done(action, rid=None, unit=None): + """Check whether broker action has completed yet. + + @param action: name of action to be performed + @returns True if action complete otherwise False + """ + rdata = relation_get(rid, unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + if not broker_rsp: + return False + + rsp = CephBrokerRsp(broker_rsp) + unit_name = local_unit().partition('/')[2] + key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) + kvstore = kv() + val = kvstore.get(key=key) + if val and val == rsp.request_id: + return True + + return False + + +def mark_broker_action_done(action, rid=None, unit=None): + """Mark action as having been completed. + + @param action: name of action to be performed + @returns None + """ + rdata = relation_get(rid, unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + if not broker_rsp: + return + + rsp = CephBrokerRsp(broker_rsp) + unit_name = local_unit().partition('/')[2] + key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) + kvstore = kv() + kvstore.set(key=key, value=rsp.request_id) + kvstore.flush() + + class CephConfContext(object): """Ceph config (ceph.conf) context. @@ -1330,7 +1372,7 @@ class CephConfContext(object): return {} conf = config_flags_parser(conf) - if type(conf) != dict: + if not isinstance(conf, dict): log("Provided config-flags is not a dictionary - ignoring", level=WARNING) return {} diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index ec5e0fe9..480a6276 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -48,6 +48,13 @@ class AptLockError(Exception): pass +class GPGKeyError(Exception): + """Exception occurs when a GPG key cannot be fetched or used. The message + indicates what the problem is. + """ + pass + + class BaseFetchHandler(object): """Base class for FetchHandler implementations in fetch plugins""" @@ -77,21 +84,22 @@ module = "charmhelpers.fetch.%s" % __platform__ fetch = importlib.import_module(module) filter_installed_packages = fetch.filter_installed_packages -install = fetch.install -upgrade = fetch.upgrade -update = fetch.update -purge = fetch.purge +install = fetch.apt_install +upgrade = fetch.apt_upgrade +update = _fetch_update = fetch.apt_update +purge = fetch.apt_purge add_source = fetch.add_source if __platform__ == "ubuntu": apt_cache = fetch.apt_cache - apt_install = fetch.install - apt_update = fetch.update - apt_upgrade = fetch.upgrade - apt_purge = fetch.purge + apt_install = fetch.apt_install + apt_update = fetch.apt_update + apt_upgrade = fetch.apt_upgrade + apt_purge = fetch.apt_purge apt_mark = fetch.apt_mark apt_hold = fetch.apt_hold apt_unhold = fetch.apt_unhold + import_key = fetch.import_key get_upstream_version = fetch.get_upstream_version elif __platform__ == "centos": yum_search = fetch.yum_search @@ -135,7 +143,7 @@ def configure_sources(update=False, for source, key in zip(sources, keys): add_source(source, key) if update: - fetch.update(fatal=True) + _fetch_update(fatal=True) def install_remote(source, *args, **kwargs): diff --git a/hooks/charmhelpers/fetch/centos.py b/hooks/charmhelpers/fetch/centos.py index 604bbfb5..a91dcff0 100644 --- a/hooks/charmhelpers/fetch/centos.py +++ b/hooks/charmhelpers/fetch/centos.py @@ -132,7 +132,7 @@ def add_source(source, key=None): key_file.write(key) key_file.flush() key_file.seek(0) - subprocess.check_call(['rpm', '--import', key_file]) + subprocess.check_call(['rpm', '--import', key_file.name]) else: subprocess.check_call(['rpm', '--import', key]) diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py index 23c707b0..112a54c3 100644 --- a/hooks/charmhelpers/fetch/snap.py +++ b/hooks/charmhelpers/fetch/snap.py @@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer: https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html """ import subprocess -from os import environ +import os from time import sleep from charmhelpers.core.hookenv import log __author__ = 'Joseph Borg ' -SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +# The return code for "couldn't acquire lock" in Snap +# (hopefully this will be improved). +SNAP_NO_LOCK = 1 SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +SNAP_CHANNELS = [ + 'edge', + 'beta', + 'candidate', + 'stable', +] class CouldNotAcquireLockException(Exception): @@ -47,13 +55,17 @@ def _snap_exec(commands): while return_code is None or return_code == SNAP_NO_LOCK: try: - return_code = subprocess.check_call(['snap'] + commands, env=environ) + return_code = subprocess.check_call(['snap'] + commands, + env=os.environ) except subprocess.CalledProcessError as e: retry_count += + 1 if retry_count > SNAP_NO_LOCK_RETRY_COUNT: - raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + raise CouldNotAcquireLockException( + 'Could not aquire lock after {} attempts' + .format(SNAP_NO_LOCK_RETRY_COUNT)) return_code = e.returncode - log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + log('Snap failed to acquire lock, trying again in {} seconds.' + .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN')) sleep(SNAP_NO_LOCK_RETRY_DELAY) return return_code diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 7bc6cc7e..545348ff 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -12,29 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import OrderedDict import os +import platform +import re import six import time import subprocess - from tempfile import NamedTemporaryFile + from charmhelpers.core.host import ( lsb_release ) -from charmhelpers.core.hookenv import log -from charmhelpers.fetch import SourceConfigError +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) +from charmhelpers.fetch import SourceConfigError, GPGKeyError +PROPOSED_POCKET = ( + "# Proposed\n" + "deb http://archive.ubuntu.com/ubuntu {}-proposed main universe " + "multiverse restricted\n") +PROPOSED_PORTS_POCKET = ( + "# Proposed\n" + "deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe " + "multiverse restricted\n") +# Only supports 64bit and ppc64 at the moment. +ARCH_TO_PROPOSED_POCKET = { + 'x86_64': PROPOSED_POCKET, + 'ppc64le': PROPOSED_PORTS_POCKET, + 'aarch64': PROPOSED_PORTS_POCKET, +} +CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" +CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main """ - -PROPOSED_POCKET = """# Proposed -deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted -""" - CLOUD_ARCHIVE_POCKETS = { # Folsom 'folsom': 'precise-updates/folsom', + 'folsom/updates': 'precise-updates/folsom', 'precise-folsom': 'precise-updates/folsom', 'precise-folsom/updates': 'precise-updates/folsom', 'precise-updates/folsom': 'precise-updates/folsom', @@ -43,6 +61,7 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/folsom': 'precise-proposed/folsom', # Grizzly 'grizzly': 'precise-updates/grizzly', + 'grizzly/updates': 'precise-updates/grizzly', 'precise-grizzly': 'precise-updates/grizzly', 'precise-grizzly/updates': 'precise-updates/grizzly', 'precise-updates/grizzly': 'precise-updates/grizzly', @@ -51,6 +70,7 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/grizzly': 'precise-proposed/grizzly', # Havana 'havana': 'precise-updates/havana', + 'havana/updates': 'precise-updates/havana', 'precise-havana': 'precise-updates/havana', 'precise-havana/updates': 'precise-updates/havana', 'precise-updates/havana': 'precise-updates/havana', @@ -59,6 +79,7 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/havana': 'precise-proposed/havana', # Icehouse 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', 'precise-icehouse': 'precise-updates/icehouse', 'precise-icehouse/updates': 'precise-updates/icehouse', 'precise-updates/icehouse': 'precise-updates/icehouse', @@ -67,6 +88,7 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/icehouse': 'precise-proposed/icehouse', # Juno 'juno': 'trusty-updates/juno', + 'juno/updates': 'trusty-updates/juno', 'trusty-juno': 'trusty-updates/juno', 'trusty-juno/updates': 'trusty-updates/juno', 'trusty-updates/juno': 'trusty-updates/juno', @@ -75,6 +97,7 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-proposed/juno': 'trusty-proposed/juno', # Kilo 'kilo': 'trusty-updates/kilo', + 'kilo/updates': 'trusty-updates/kilo', 'trusty-kilo': 'trusty-updates/kilo', 'trusty-kilo/updates': 'trusty-updates/kilo', 'trusty-updates/kilo': 'trusty-updates/kilo', @@ -83,6 +106,7 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-proposed/kilo': 'trusty-proposed/kilo', # Liberty 'liberty': 'trusty-updates/liberty', + 'liberty/updates': 'trusty-updates/liberty', 'trusty-liberty': 'trusty-updates/liberty', 'trusty-liberty/updates': 'trusty-updates/liberty', 'trusty-updates/liberty': 'trusty-updates/liberty', @@ -91,6 +115,7 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-proposed/liberty': 'trusty-proposed/liberty', # Mitaka 'mitaka': 'trusty-updates/mitaka', + 'mitaka/updates': 'trusty-updates/mitaka', 'trusty-mitaka': 'trusty-updates/mitaka', 'trusty-mitaka/updates': 'trusty-updates/mitaka', 'trusty-updates/mitaka': 'trusty-updates/mitaka', @@ -99,6 +124,7 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', # Newton 'newton': 'xenial-updates/newton', + 'newton/updates': 'xenial-updates/newton', 'xenial-newton': 'xenial-updates/newton', 'xenial-newton/updates': 'xenial-updates/newton', 'xenial-updates/newton': 'xenial-updates/newton', @@ -107,12 +133,13 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-proposed/newton': 'xenial-proposed/newton', # Ocata 'ocata': 'xenial-updates/ocata', + 'ocata/updates': 'xenial-updates/ocata', 'xenial-ocata': 'xenial-updates/ocata', 'xenial-ocata/updates': 'xenial-updates/ocata', 'xenial-updates/ocata': 'xenial-updates/ocata', 'ocata/proposed': 'xenial-proposed/ocata', 'xenial-ocata/proposed': 'xenial-proposed/ocata', - 'xenial-ocata/newton': 'xenial-proposed/ocata', + 'xenial-proposed/ocata': 'xenial-proposed/ocata', # Pike 'pike': 'xenial-updates/pike', 'xenial-pike': 'xenial-updates/pike', @@ -120,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/pike': 'xenial-updates/pike', 'pike/proposed': 'xenial-proposed/pike', 'xenial-pike/proposed': 'xenial-proposed/pike', - 'xenial-pike/newton': 'xenial-proposed/pike', + 'xenial-proposed/pike': 'xenial-proposed/pike', # Queens 'queens': 'xenial-updates/queens', 'xenial-queens': 'xenial-updates/queens', @@ -128,12 +155,13 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/queens': 'xenial-updates/queens', 'queens/proposed': 'xenial-proposed/queens', 'xenial-queens/proposed': 'xenial-proposed/queens', - 'xenial-queens/newton': 'xenial-proposed/queens', + 'xenial-proposed/queens': 'xenial-proposed/queens', } + APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. +CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -161,7 +189,7 @@ def apt_cache(in_memory=True, progress=None): return apt_pkg.Cache(progress) -def install(packages, options=None, fatal=False): +def apt_install(packages, options=None, fatal=False): """Install one or more packages.""" if options is None: options = ['--option=Dpkg::Options::=--force-confold'] @@ -178,7 +206,7 @@ def install(packages, options=None, fatal=False): _run_apt_command(cmd, fatal) -def upgrade(options=None, fatal=False, dist=False): +def apt_upgrade(options=None, fatal=False, dist=False): """Upgrade all packages.""" if options is None: options = ['--option=Dpkg::Options::=--force-confold'] @@ -193,13 +221,13 @@ def upgrade(options=None, fatal=False, dist=False): _run_apt_command(cmd, fatal) -def update(fatal=False): +def apt_update(fatal=False): """Update local apt cache.""" cmd = ['apt-get', 'update'] _run_apt_command(cmd, fatal) -def purge(packages, fatal=False): +def apt_purge(packages, fatal=False): """Purge one or more packages.""" cmd = ['apt-get', '--assume-yes', 'purge'] if isinstance(packages, six.string_types): @@ -233,7 +261,45 @@ def apt_unhold(packages, fatal=False): return apt_mark(packages, 'unhold', fatal=fatal) -def add_source(source, key=None): +def import_key(keyid): + """Import a key in either ASCII Armor or Radix64 format. + + `keyid` is either the keyid to fetch from a PGP server, or + the key in ASCII armor foramt. + + :param keyid: String of key (or key id). + :raises: GPGKeyError if the key could not be imported + """ + key = keyid.strip() + if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and + key.endswith('-----END PGP PUBLIC KEY BLOCK-----')): + log("PGP key found (looks like ASCII Armor format)", level=DEBUG) + log("Importing ASCII Armor PGP key", level=DEBUG) + with NamedTemporaryFile() as keyfile: + with open(keyfile.name, 'w') as fd: + fd.write(key) + fd.write("\n") + cmd = ['apt-key', 'add', keyfile.name] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + error = "Error importing PGP key '{}'".format(key) + log(error) + raise GPGKeyError(error) + else: + log("PGP key found (looks like Radix64 format)", level=DEBUG) + log("Importing PGP key from keyserver", level=DEBUG) + cmd = ['apt-key', 'adv', '--keyserver', + 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + error = "Error importing PGP key '{}'".format(key) + log(error) + raise GPGKeyError(error) + + +def add_source(source, key=None, fail_invalid=False): """Add a package source to this system. @param source: a URL or sources.list entry, as supported by @@ -249,6 +315,33 @@ def add_source(source, key=None): such as 'cloud:icehouse' 'distro' may be used as a noop + Full list of source specifications supported by the function are: + + 'distro': A NOP; i.e. it has no effect. + 'proposed': the proposed deb spec [2] is wrtten to + /etc/apt/sources.list/proposed + 'distro-proposed': adds -proposed to the debs [2] + 'ppa:': add-apt-repository --yes + 'deb ': add-apt-repository --yes deb + 'http://....': add-apt-repository --yes http://... + 'cloud-archive:': add-apt-repository -yes cloud-archive: + 'cloud:[-staging]': specify a Cloud Archive pocket with + optional staging version. If staging is used then the staging PPA [2] + with be used. If staging is NOT used then the cloud archive [3] will be + added, and the 'ubuntu-cloud-keyring' package will be added for the + current distro. + + Otherwise the source is not recognised and this is logged to the juju log. + However, no error is raised, unless sys_error_on_exit is True. + + [1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main + where {} is replaced with the derived pocket name. + [2] deb http://archive.ubuntu.com/ubuntu {}-proposed \ + main universe multiverse restricted + where {} is replaced with the lsb_release codename (e.g. xenial) + [3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu + to /etc/apt/sources.list.d/cloud-archive-list + @param key: A key to be added to the system's APT keyring and used to verify the signatures on packages. Ideally, this should be an ASCII format GPG public key including the block headers. A GPG key @@ -256,51 +349,142 @@ def add_source(source, key=None): available to retrieve the actual public key from a public keyserver placing your Juju environment at risk. ppa and cloud archive keys are securely added automtically, so sould not be provided. + + @param fail_invalid: (boolean) if True, then the function raises a + SourceConfigError is there is no matching installation source. + + @raises SourceConfigError() if for cloud:, the is not a + valid pocket in CLOUD_ARCHIVE_POCKETS """ + _mapping = OrderedDict([ + (r"^distro$", lambda: None), # This is a NOP + (r"^(?:proposed|distro-proposed)$", _add_proposed), + (r"^cloud-archive:(.*)$", _add_apt_repository), + (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository), + (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), + (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), + (r"^cloud:(.*)$", _add_cloud_pocket), + (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), + ]) if source is None: - log('Source is not present. Skipping') - return - - if (source.startswith('ppa:') or - source.startswith('http') or - source.startswith('deb ') or - source.startswith('cloud-archive:')): - cmd = ['add-apt-repository', '--yes', source] - _run_with_retries(cmd) - elif source.startswith('cloud:'): - install(filter_installed_packages(['ubuntu-cloud-keyring']), - fatal=True) - pocket = source.split(':')[-1] - if pocket not in CLOUD_ARCHIVE_POCKETS: - raise SourceConfigError( - 'Unsupported cloud: source option %s' % - pocket) - actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] - with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: - apt.write(CLOUD_ARCHIVE.format(actual_pocket)) - elif source == 'proposed': - release = lsb_release()['DISTRIB_CODENAME'] - with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: - apt.write(PROPOSED_POCKET.format(release)) - elif source == 'distro': - pass + source = '' + for r, fn in six.iteritems(_mapping): + m = re.match(r, source) + if m: + # call the assoicated function with the captured groups + # raises SourceConfigError on error. + fn(*m.groups()) + if key: + try: + import_key(key) + except GPGKeyError as e: + raise SourceConfigError(str(e)) + break else: - log("Unknown source: {!r}".format(source)) + # nothing matched. log an error and maybe sys.exit + err = "Unknown source: {!r}".format(source) + log(err) + if fail_invalid: + raise SourceConfigError(err) - if key: - if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: - with NamedTemporaryFile('w+') as key_file: - key_file.write(key) - key_file.flush() - key_file.seek(0) - subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file) - else: - # Note that hkp: is in no way a secure protocol. Using a - # GPG key id is pointless from a security POV unless you - # absolutely trust your network and DNS. - subprocess.check_call(['apt-key', 'adv', '--keyserver', - 'hkp://keyserver.ubuntu.com:80', '--recv', - key]) + +def _add_proposed(): + """Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list + + Uses lsb_release()['DISTRIB_CODENAME'] to determine the correct staza for + the deb line. + + For intel architecutres PROPOSED_POCKET is used for the release, but for + other architectures PROPOSED_PORTS_POCKET is used for the release. + """ + release = lsb_release()['DISTRIB_CODENAME'] + arch = platform.machine() + if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET): + raise SourceConfigError("Arch {} not supported for (distro-)proposed" + .format(arch)) + with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: + apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release)) + + +def _add_apt_repository(spec): + """Add the spec using add_apt_repository + + :param spec: the parameter to pass to add_apt_repository + """ + _run_with_retries(['add-apt-repository', '--yes', spec]) + + +def _add_cloud_pocket(pocket): + """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list + + Note that this overwrites the existing file if there is one. + + This function also converts the simple pocket in to the actual pocket using + the CLOUD_ARCHIVE_POCKETS mapping. + + :param pocket: string representing the pocket to add a deb spec for. + :raises: SourceConfigError if the cloud pocket doesn't exist or the + requested release doesn't match the current distro version. + """ + apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), + fatal=True) + if pocket not in CLOUD_ARCHIVE_POCKETS: + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) + actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] + with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: + apt.write(CLOUD_ARCHIVE.format(actual_pocket)) + + +def _add_cloud_staging(cloud_archive_release, openstack_release): + """Add the cloud staging repository which is in + ppa:ubuntu-cloud-archive/-staging + + This function checks that the cloud_archive_release matches the current + codename for the distro that charm is being installed on. + + :param cloud_archive_release: string, codename for the release. + :param openstack_release: String, codename for the openstack release. + :raises: SourceConfigError if the cloud_archive_release doesn't match the + current version of the os. + """ + _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) + ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release) + cmd = 'add-apt-repository -y {}'.format(ppa) + _run_with_retries(cmd.split(' ')) + + +def _add_cloud_distro_check(cloud_archive_release, openstack_release): + """Add the cloud pocket, but also check the cloud_archive_release against + the current distro, and use the openstack_release as the full lookup. + + This just calls _add_cloud_pocket() with the openstack_release as pocket + to get the correct cloud-archive.list for dpkg to work with. + + :param cloud_archive_release:String, codename for the distro release. + :param openstack_release: String, spec for the release to look up in the + CLOUD_ARCHIVE_POCKETS + :raises: SourceConfigError if this is the wrong distro, or the pocket spec + doesn't exist. + """ + _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) + _add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release)) + + +def _verify_is_ubuntu_rel(release, os_release): + """Verify that the release is in the same as the current ubuntu release. + + :param release: String, lowercase for the release. + :param os_release: String, the os_release being asked for + :raises: SourceConfigError if the release is not the same as the ubuntu + release. + """ + ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] + if release != ubuntu_rel: + raise SourceConfigError( + 'Invalid Cloud Archive release specified: {}-{} on this Ubuntu' + 'version ({})'.format(release, os_release, ubuntu_rel)) def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), @@ -316,9 +500,12 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), :param: cmd_env: dict: Environment variables to add to the command run. """ - env = os.environ.copy() + env = None + kwargs = {} if cmd_env: + env = os.environ.copy() env.update(cmd_env) + kwargs['env'] = env if not retry_message: retry_message = "Failed executing '{}'".format(" ".join(cmd)) @@ -330,7 +517,8 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), retry_results = (None,) + retry_exitcodes while result in retry_results: try: - result = subprocess.check_call(cmd, env=env) + # result = subprocess.check_call(cmd, env=env) + result = subprocess.check_call(cmd, **kwargs) except subprocess.CalledProcessError as e: retry_count = retry_count + 1 if retry_count > max_retries: @@ -343,6 +531,7 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), def _run_apt_command(cmd, fatal=False): """Run an apt command with optional retries. + :param: cmd: str: The apt command to run. :param: fatal: bool: Whether the command's output should be checked and retried. """ diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 59a3ad8c..fdf0deab 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -212,7 +212,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): # Authenticate admin with neutron ep = self.keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') self.neutron = neutronclient.Client(auth_url=ep, username='admin', password='openstack', @@ -686,7 +686,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'quantum-network-service', 'neutron-gateway:quantum-network-service') ep = self.keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') conf = '/etc/neutron/l3_agent.ini' expected = { @@ -756,7 +756,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'config file data...') unit = self.neutron_gateway_sentry ep = self.keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') nova_cc_relation = self.nova_cc_sentry.relation( 'quantum-network-service', 'neutron-gateway:quantum-network-service') @@ -817,7 +817,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'quantum-network-service', 'neutron-gateway:quantum-network-service') ep = self.keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') expected = { 'DEFAULT': { diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py index 48867880..e7aa4715 100644 --- a/tests/charmhelpers/__init__.py +++ b/tests/charmhelpers/__init__.py @@ -14,6 +14,11 @@ # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. +from __future__ import print_function +from __future__ import absolute_import + +import functools +import inspect import subprocess import sys @@ -34,3 +39,59 @@ except ImportError: else: subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) import yaml # flake8: noqa + + +# Holds a list of mapping of mangled function names that have been deprecated +# using the @deprecate decorator below. This is so that the warning is only +# printed once for each usage of the function. +__deprecated_functions = {} + + +def deprecate(warning, date=None, log=None): + """Add a deprecation warning the first time the function is used. + The date, which is a string in semi-ISO8660 format indicate the year-month + that the function is officially going to be removed. + + usage: + + @deprecate('use core/fetch/add_source() instead', '2017-04') + def contributed_add_source_thing(...): + ... + + And it then prints to the log ONCE that the function is deprecated. + The reason for passing the logging function (log) is so that hookenv.log + can be used for a charm if needed. + + :param warning: String to indicat where it has moved ot. + :param date: optional sting, in YYYY-MM format to indicate when the + function will definitely (probably) be removed. + :param log: The log function to call to log. If not, logs to stdout + """ + def wrap(f): + + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + try: + module = inspect.getmodule(f) + file = inspect.getsourcefile(f) + lines = inspect.getsourcelines(f) + f_name = "{}-{}-{}..{}-{}".format( + module.__name__, file, lines[0], lines[-1], f.__name__) + except (IOError, TypeError): + # assume it was local, so just use the name of the function + f_name = f.__name__ + if f_name not in __deprecated_functions: + __deprecated_functions[f_name] = True + s = "DEPRECATION WARNING: Function {} is being removed".format( + f.__name__) + if date: + s = "{} on/around {}".format(s, date) + if warning: + s = "{} : {}".format(s, warning) + if log: + log(s) + else: + print(s) + return f(*args, **kwargs) + return wrapped_f + return wrap diff --git a/tests/charmhelpers/contrib/network/ip.py b/tests/charmhelpers/contrib/network/ip.py index fc3f5e3e..d7e6debf 100644 --- a/tests/charmhelpers/contrib/network/ip.py +++ b/tests/charmhelpers/contrib/network/ip.py @@ -243,11 +243,13 @@ def is_ipv6_disabled(): try: result = subprocess.check_output( ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], - stderr=subprocess.STDOUT) - return "net.ipv6.conf.all.disable_ipv6 = 1" in result + stderr=subprocess.STDOUT, + universal_newlines=True) except subprocess.CalledProcessError: return True + return "net.ipv6.conf.all.disable_ipv6 = 1" in result + def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None): diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index bcef4cd0..c8edbf65 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -25,9 +25,12 @@ import urlparse import cinderclient.v1.client as cinder_client import glanceclient.v1.client as glance_client import heatclient.v1.client as heat_client -import keystoneclient.v2_0 as keystone_client -from keystoneclient.auth.identity import v3 as keystone_id_v3 -from keystoneclient import session as keystone_session +from keystoneclient.v2_0 import client as keystone_client +from keystoneauth1.identity import ( + v3, + v2, +) +from keystoneauth1 import session as keystone_session from keystoneclient.v3 import client as keystone_client_v3 from novaclient import exceptions @@ -368,12 +371,20 @@ class OpenStackAmuletUtils(AmuletUtils): port) if not api_version or api_version == 2: ep = base_ep + "/v2.0" - return keystone_client.Client(username=username, password=password, - tenant_name=project_name, - auth_url=ep) + auth = v2.Password( + username=username, + password=password, + tenant_name=project_name, + auth_url=ep + ) + sess = keystone_session.Session(auth=auth) + client = keystone_client.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client else: ep = base_ep + "/v3" - auth = keystone_id_v3.Password( + auth = v3.Password( user_domain_name=user_domain_name, username=username, password=password, @@ -382,36 +393,45 @@ class OpenStackAmuletUtils(AmuletUtils): project_name=project_name, auth_url=ep ) - return keystone_client_v3.Client( - session=keystone_session.Session(auth=auth) - ) + sess = keystone_session.Session(auth=auth) + client = keystone_client_v3.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client def authenticate_keystone_admin(self, keystone_sentry, user, password, tenant=None, api_version=None, - keystone_ip=None): + keystone_ip=None, user_domain_name=None, + project_domain_name=None, + project_name=None): """Authenticates admin user with the keystone admin endpoint.""" self.log.debug('Authenticating keystone admin...') if not keystone_ip: keystone_ip = keystone_sentry.info['public-address'] - user_domain_name = None - domain_name = None - if api_version == 3: + # To support backward compatibility usage of this function + if not project_name: + project_name = tenant + if api_version == 3 and not user_domain_name: user_domain_name = 'admin_domain' - domain_name = user_domain_name + if api_version == 3 and not project_domain_name: + project_domain_name = 'admin_domain' + if api_version == 3 and not project_name: + project_name = 'admin' - return self.authenticate_keystone(keystone_ip, user, password, - project_name=tenant, - api_version=api_version, - user_domain_name=user_domain_name, - domain_name=domain_name, - admin_port=True) + return self.authenticate_keystone( + keystone_ip, user, password, + api_version=api_version, + user_domain_name=user_domain_name, + project_domain_name=project_domain_name, + project_name=project_name, + admin_port=True) def authenticate_keystone_user(self, keystone, user, password, tenant): """Authenticates a regular user with the keystone public endpoint.""" self.log.debug('Authenticating keystone user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') + interface='publicURL') keystone_ip = urlparse.urlparse(ep).hostname return self.authenticate_keystone(keystone_ip, user, password, @@ -421,22 +441,32 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates admin user with glance.""" self.log.debug('Authenticating glance admin...') ep = keystone.service_catalog.url_for(service_type='image', - endpoint_type='adminURL') - return glance_client.Client(ep, token=keystone.auth_token) + interface='adminURL') + if keystone.session: + return glance_client.Client(ep, session=keystone.session) + else: + return glance_client.Client(ep, token=keystone.auth_token) def authenticate_heat_admin(self, keystone): """Authenticates the admin user with heat.""" self.log.debug('Authenticating heat admin...') ep = keystone.service_catalog.url_for(service_type='orchestration', - endpoint_type='publicURL') - return heat_client.Client(endpoint=ep, token=keystone.auth_token) + interface='publicURL') + if keystone.session: + return heat_client.Client(endpoint=ep, session=keystone.session) + else: + return heat_client.Client(endpoint=ep, token=keystone.auth_token) def authenticate_nova_user(self, keystone, user, password, tenant): """Authenticates a regular user with nova-api.""" self.log.debug('Authenticating nova user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - if novaclient.__version__[0] >= "7": + interface='publicURL') + if keystone.session: + return nova_client.Client(NOVA_CLIENT_VERSION, + session=keystone.session, + auth_url=ep) + elif novaclient.__version__[0] >= "7": return nova_client.Client(NOVA_CLIENT_VERSION, username=user, password=password, project_name=tenant, auth_url=ep) @@ -449,12 +479,15 @@ class OpenStackAmuletUtils(AmuletUtils): """Authenticates a regular user with swift api.""" self.log.debug('Authenticating swift user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', - endpoint_type='publicURL') - return swiftclient.Connection(authurl=ep, - user=user, - key=password, - tenant_name=tenant, - auth_version='2.0') + interface='publicURL') + if keystone.session: + return swiftclient.Connection(session=keystone.session) + else: + return swiftclient.Connection(authurl=ep, + user=user, + key=password, + tenant_name=tenant, + auth_version='2.0') def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto", ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True): diff --git a/tests/charmhelpers/contrib/openstack/utils.py b/tests/charmhelpers/contrib/openstack/utils.py index 161c786b..d15a6317 100644 --- a/tests/charmhelpers/contrib/openstack/utils.py +++ b/tests/charmhelpers/contrib/openstack/utils.py @@ -26,11 +26,12 @@ import functools import shutil import six -import tempfile import traceback import uuid import yaml +from charmhelpers import deprecate + from charmhelpers.contrib.network import ip from charmhelpers.core import unitdata @@ -41,7 +42,6 @@ from charmhelpers.core.hookenv import ( config, log as juju_log, charm_dir, - DEBUG, INFO, ERROR, related_units, @@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import ( status_set, hook_name, application_version_set, + cached, ) from charmhelpers.core.strutils import BasicStringComparator @@ -82,11 +83,21 @@ from charmhelpers.core.host import ( restart_on_change_helper, ) from charmhelpers.fetch import ( - apt_install, apt_cache, install_remote, + import_key as fetch_import_key, + add_source as fetch_add_source, + SourceConfigError, + GPGKeyError, get_upstream_version ) + +from charmhelpers.fetch.snap import ( + snap_install, + snap_refresh, + SNAP_CHANNELS, +) + from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device from charmhelpers.contrib.openstack.exceptions import OSContextError @@ -324,8 +335,10 @@ def get_os_codename_install_source(src): return ca_rel # Best guess match based on deb string provided - if src.startswith('deb') or src.startswith('ppa'): - for k, v in six.iteritems(OPENSTACK_CODENAMES): + if (src.startswith('deb') or + src.startswith('ppa') or + src.startswith('snap')): + for v in OPENSTACK_CODENAMES.values(): if v in src: return v @@ -394,6 +407,19 @@ def get_swift_codename(version): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + + if snap_install_requested(): + cmd = ['snap', 'list', package] + try: + out = subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + return None + lines = out.split('\n') + for line in lines: + if package in line: + # Second item in list is Version + return line.split()[1] + import apt_pkg as apt cache = apt_cache() @@ -469,13 +495,14 @@ def get_os_version_package(pkg, fatal=True): # error_out(e) -os_rel = None +# Module local cache variable for the os_release. +_os_rel = None def reset_os_release(): '''Unset the cached os_release version''' - global os_rel - os_rel = None + global _os_rel + _os_rel = None def os_release(package, base='essex', reset_cache=False): @@ -489,150 +516,77 @@ def os_release(package, base='essex', reset_cache=False): the installation source, the earliest release supported by the charm should be returned. ''' - global os_rel + global _os_rel if reset_cache: reset_os_release() - if os_rel: - return os_rel - os_rel = (git_os_codename_install_source(config('openstack-origin-git')) or - get_os_codename_package(package, fatal=False) or - get_os_codename_install_source(config('openstack-origin')) or - base) - return os_rel + if _os_rel: + return _os_rel + _os_rel = ( + git_os_codename_install_source(config('openstack-origin-git')) or + get_os_codename_package(package, fatal=False) or + get_os_codename_install_source(config('openstack-origin')) or + base) + return _os_rel +@deprecate("moved to charmhelpers.fetch.import_key()", "2017-07", log=juju_log) def import_key(keyid): - key = keyid.strip() - if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and - key.endswith('-----END PGP PUBLIC KEY BLOCK-----')): - juju_log("PGP key found (looks like ASCII Armor format)", level=DEBUG) - juju_log("Importing ASCII Armor PGP key", level=DEBUG) - with tempfile.NamedTemporaryFile() as keyfile: - with open(keyfile.name, 'w') as fd: - fd.write(key) - fd.write("\n") + """Import a key, either ASCII armored, or a GPG key id. - cmd = ['apt-key', 'add', keyfile.name] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - error_out("Error importing PGP key '%s'" % key) - else: - juju_log("PGP key found (looks like Radix64 format)", level=DEBUG) - juju_log("Importing PGP key from keyserver", level=DEBUG) - cmd = ['apt-key', 'adv', '--keyserver', - 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - error_out("Error importing PGP key '%s'" % key) + @param keyid: the key in ASCII armor format, or a GPG key id. + @raises SystemExit() via sys.exit() on failure. + """ + try: + return fetch_import_key(keyid) + except GPGKeyError as e: + error_out("Could not import key: {}".format(str(e))) -def get_source_and_pgp_key(input): - """Look for a pgp key ID or ascii-armor key in the given input.""" - index = input.strip() - index = input.rfind('|') - if index < 0: - return input, None +def get_source_and_pgp_key(source_and_key): + """Look for a pgp key ID or ascii-armor key in the given input. - key = input[index + 1:].strip('|') - source = input[:index] - return source, key + :param source_and_key: Sting, "source_spec|keyid" where '|keyid' is + optional. + :returns (source_spec, key_id OR None) as a tuple. Returns None for key_id + if there was no '|' in the source_and_key string. + """ + try: + source, key = source_and_key.split('|', 2) + return source, key or None + except ValueError: + return source_and_key, None -def configure_installation_source(rel): - '''Configure apt installation source.''' - if rel == 'distro': - return - elif rel == 'distro-proposed': - ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] - with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: - f.write(DISTRO_PROPOSED % ubuntu_rel) - elif rel[:4] == "ppa:": - src, key = get_source_and_pgp_key(rel) - if key: - import_key(key) +@deprecate("use charmhelpers.fetch.add_source() instead.", + "2017-07", log=juju_log) +def configure_installation_source(source_plus_key): + """Configure an installation source. - subprocess.check_call(["add-apt-repository", "-y", src]) - elif rel[:3] == "deb": - src, key = get_source_and_pgp_key(rel) - if key: - import_key(key) + The functionality is provided by charmhelpers.fetch.add_source() + The difference between the two functions is that add_source() signature + requires the key to be passed directly, whereas this function passes an + optional key by appending '|' to the end of the source specificiation + 'source'. - with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: - f.write(src) - elif rel[:6] == 'cloud:': - ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] - rel = rel.split(':')[1] - u_rel = rel.split('-')[0] - ca_rel = rel.split('-')[1] + Another difference from add_source() is that the function calls sys.exit(1) + if the configuration fails, whereas add_source() raises + SourceConfigurationError(). Another difference, is that add_source() + silently fails (with a juju_log command) if there is no matching source to + configure, whereas this function fails with a sys.exit(1) - if u_rel != ubuntu_rel: - e = 'Cannot install from Cloud Archive pocket %s on this Ubuntu '\ - 'version (%s)' % (ca_rel, ubuntu_rel) - error_out(e) + :param source: String_plus_key -- see above for details. - if 'staging' in ca_rel: - # staging is just a regular PPA. - os_rel = ca_rel.split('/')[0] - ppa = 'ppa:ubuntu-cloud-archive/%s-staging' % os_rel - cmd = 'add-apt-repository -y %s' % ppa - subprocess.check_call(cmd.split(' ')) - return + Note that the behaviour on error is to log the error to the juju log and + then call sys.exit(1). + """ + # extract the key if there is one, denoted by a '|' in the rel + source, key = get_source_and_pgp_key(source_plus_key) - # map charm config options to actual archive pockets. - pockets = { - 'folsom': 'precise-updates/folsom', - 'folsom/updates': 'precise-updates/folsom', - 'folsom/proposed': 'precise-proposed/folsom', - 'grizzly': 'precise-updates/grizzly', - 'grizzly/updates': 'precise-updates/grizzly', - 'grizzly/proposed': 'precise-proposed/grizzly', - 'havana': 'precise-updates/havana', - 'havana/updates': 'precise-updates/havana', - 'havana/proposed': 'precise-proposed/havana', - 'icehouse': 'precise-updates/icehouse', - 'icehouse/updates': 'precise-updates/icehouse', - 'icehouse/proposed': 'precise-proposed/icehouse', - 'juno': 'trusty-updates/juno', - 'juno/updates': 'trusty-updates/juno', - 'juno/proposed': 'trusty-proposed/juno', - 'kilo': 'trusty-updates/kilo', - 'kilo/updates': 'trusty-updates/kilo', - 'kilo/proposed': 'trusty-proposed/kilo', - 'liberty': 'trusty-updates/liberty', - 'liberty/updates': 'trusty-updates/liberty', - 'liberty/proposed': 'trusty-proposed/liberty', - 'mitaka': 'trusty-updates/mitaka', - 'mitaka/updates': 'trusty-updates/mitaka', - 'mitaka/proposed': 'trusty-proposed/mitaka', - 'newton': 'xenial-updates/newton', - 'newton/updates': 'xenial-updates/newton', - 'newton/proposed': 'xenial-proposed/newton', - 'ocata': 'xenial-updates/ocata', - 'ocata/updates': 'xenial-updates/ocata', - 'ocata/proposed': 'xenial-proposed/ocata', - 'pike': 'xenial-updates/pike', - 'pike/updates': 'xenial-updates/pike', - 'pike/proposed': 'xenial-proposed/pike', - 'queens': 'xenial-updates/queens', - 'queens/updates': 'xenial-updates/queens', - 'queens/proposed': 'xenial-proposed/queens', - } - - try: - pocket = pockets[ca_rel] - except KeyError: - e = 'Invalid Cloud Archive release specified: %s' % rel - error_out(e) - - src = "deb %s %s main" % (CLOUD_ARCHIVE_URL, pocket) - apt_install('ubuntu-cloud-keyring', fatal=True) - - with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as f: - f.write(src) - else: - error_out("Invalid openstack-release specified: %s" % rel) + # handle the ordinary sources via add_source + try: + fetch_add_source(source, key, fail_invalid=True) + except SourceConfigError as se: + error_out(str(se)) def config_value_changed(option): @@ -677,12 +631,14 @@ def openstack_upgrade_available(package): :returns: bool: : Returns True if configured installation source offers a newer version of package. - """ import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) + if not cur_vers: + # The package has not been installed yet do not attempt upgrade + return False if "swift" in package: codename = get_os_codename_install_source(src) avail_vers = get_os_version_codename_swift(codename) @@ -1933,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False, return wrap +def ordered(orderme): + """Converts the provided dictionary into a collections.OrderedDict. + + The items in the returned OrderedDict will be inserted based on the + natural sort order of the keys. Nested dictionaries will also be sorted + in order to ensure fully predictable ordering. + + :param orderme: the dict to order + :return: collections.OrderedDict + :raises: ValueError: if `orderme` isn't a dict instance. + """ + if not isinstance(orderme, dict): + raise ValueError('argument must be a dict type') + + result = OrderedDict() + for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]): + if isinstance(v, dict): + result[k] = ordered(v) + else: + result[k] = v + + return result + + def config_flags_parser(config_flags): """Parses config flags string into dict. @@ -1944,15 +1924,13 @@ def config_flags_parser(config_flags): example, a string in the format of 'key1=value1, key2=value2' will return a dict of: - {'key1': 'value1', - 'key2': 'value2'}. + {'key1': 'value1', 'key2': 'value2'}. 2. A string in the above format, but supporting a comma-delimited list of values for the same key. For example, a string in the format of 'key1=value1, key2=value3,value4,value5' will return a dict of: - {'key1', 'value1', - 'key2', 'value2,value3,value4'} + {'key1': 'value1', 'key2': 'value2,value3,value4'} 3. A string containing a colon character (:) prior to an equal character (=) will be treated as yaml and parsed as such. This can be @@ -1972,7 +1950,7 @@ def config_flags_parser(config_flags): equals = config_flags.find('=') if colon > 0: if colon < equals or equals < 0: - return yaml.safe_load(config_flags) + return ordered(yaml.safe_load(config_flags)) if config_flags.find('==') >= 0: juju_log("config_flags is not in expected format (key=value)", @@ -1985,7 +1963,7 @@ def config_flags_parser(config_flags): # split on '='. split = config_flags.strip(' =').split('=') limit = len(split) - flags = {} + flags = OrderedDict() for i in range(0, limit - 1): current = split[i] next = split[i + 1] @@ -2052,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None): if enable_memcache(source=source, release=release): packages.extend(['memcached', 'python-memcache']) return packages + + +def update_json_file(filename, items): + """Updates the json `filename` with a given dict. + :param filename: json filename (i.e.: /etc/glance/policy.json) + :param items: dict of items to update + """ + with open(filename) as fd: + policy = json.load(fd) + policy.update(items) + with open(filename, "w") as fd: + fd.write(json.dumps(policy, indent=4)) + + +@cached +def snap_install_requested(): + """ Determine if installing from snaps + + If openstack-origin is of the form snap:channel-series-release + and channel is in SNAPS_CHANNELS return True. + """ + origin = config('openstack-origin') + if not origin.startswith('snap:'): + return False + + _src = origin[5:] + channel, series, release = _src.split('-') + if channel.lower() in SNAP_CHANNELS: + return True + return False + + +def get_snaps_install_info_from_origin(snaps, src, mode='classic'): + """Generate a dictionary of snap install information from origin + + @param snaps: List of snaps + @param src: String of openstack-origin or source of the form + snap:channel-series-track + @param mode: String classic, devmode or jailmode + @returns: Dictionary of snaps with channels and modes + """ + + if not src.startswith('snap:'): + juju_log("Snap source is not a snap origin", 'WARN') + return {} + + _src = src[5:] + _channel, _series, _release = _src.split('-') + channel = '--channel={}/{}'.format(_release, _channel) + + return {snap: {'channel': channel, 'mode': mode} + for snap in snaps} + + +def install_os_snaps(snaps, refresh=False): + """Install OpenStack snaps from channel and with mode + + @param snaps: Dictionary of snaps with channels and modes of the form: + {'snap_name': {'channel': 'snap_channel', + 'mode': 'snap_mode'}} + Where channel a snapstore channel and mode is --classic, --devmode or + --jailmode. + @param post_snap_install: Callback function to run after snaps have been + installed + """ + + def _ensure_flag(flag): + if flag.startswith('--'): + return flag + return '--{}'.format(flag) + + if refresh: + for snap in snaps.keys(): + snap_refresh(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) + else: + for snap in snaps.keys(): + snap_install(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) diff --git a/tests/charmhelpers/contrib/storage/linux/bcache.py b/tests/charmhelpers/contrib/storage/linux/bcache.py new file mode 100644 index 00000000..605991e1 --- /dev/null +++ b/tests/charmhelpers/contrib/storage/linux/bcache.py @@ -0,0 +1,74 @@ +# Copyright 2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import json + +from charmhelpers.core.hookenv import log + +stats_intervals = ['stats_day', 'stats_five_minute', + 'stats_hour', 'stats_total'] + +SYSFS = '/sys' + + +class Bcache(object): + """Bcache behaviour + """ + + def __init__(self, cachepath): + self.cachepath = cachepath + + @classmethod + def fromdevice(cls, devname): + return cls('{}/block/{}/bcache'.format(SYSFS, devname)) + + def __str__(self): + return self.cachepath + + def get_stats(self, interval): + """Get cache stats + """ + intervaldir = 'stats_{}'.format(interval) + path = "{}/{}".format(self.cachepath, intervaldir) + out = dict() + for elem in os.listdir(path): + out[elem] = open('{}/{}'.format(path, elem)).read().strip() + return out + + +def get_bcache_fs(): + """Return all cache sets + """ + cachesetroot = "{}/fs/bcache".format(SYSFS) + try: + dirs = os.listdir(cachesetroot) + except OSError: + log("No bcache fs found") + return [] + cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')]) + return cacheset + + +def get_stats_action(cachespec, interval): + """Action for getting bcache statistics for a given cachespec. + Cachespec can either be a device name, eg. 'sdb', which will retrieve + cache stats for the given device, or 'global', which will retrieve stats + for all cachesets + """ + if cachespec == 'global': + caches = get_bcache_fs() + else: + caches = [Bcache.fromdevice(cachespec)] + res = dict((c.cachepath, c.get_stats(interval)) for c in caches) + return json.dumps(res, indent=4, separators=(',', ': ')) diff --git a/tests/charmhelpers/contrib/storage/linux/ceph.py b/tests/charmhelpers/contrib/storage/linux/ceph.py index 9417d684..e5a01b1b 100644 --- a/tests/charmhelpers/contrib/storage/linux/ceph.py +++ b/tests/charmhelpers/contrib/storage/linux/ceph.py @@ -63,6 +63,7 @@ from charmhelpers.core.host import ( from charmhelpers.fetch import ( apt_install, ) +from charmhelpers.core.unitdata import kv from charmhelpers.core.kernel import modprobe from charmhelpers.contrib.openstack.utils import config_flags_parser @@ -1314,6 +1315,47 @@ def send_request_if_needed(request, relation='ceph'): relation_set(relation_id=rid, broker_req=request.request) +def is_broker_action_done(action, rid=None, unit=None): + """Check whether broker action has completed yet. + + @param action: name of action to be performed + @returns True if action complete otherwise False + """ + rdata = relation_get(rid, unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + if not broker_rsp: + return False + + rsp = CephBrokerRsp(broker_rsp) + unit_name = local_unit().partition('/')[2] + key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) + kvstore = kv() + val = kvstore.get(key=key) + if val and val == rsp.request_id: + return True + + return False + + +def mark_broker_action_done(action, rid=None, unit=None): + """Mark action as having been completed. + + @param action: name of action to be performed + @returns None + """ + rdata = relation_get(rid, unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + if not broker_rsp: + return + + rsp = CephBrokerRsp(broker_rsp) + unit_name = local_unit().partition('/')[2] + key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) + kvstore = kv() + kvstore.set(key=key, value=rsp.request_id) + kvstore.flush() + + class CephConfContext(object): """Ceph config (ceph.conf) context. @@ -1330,7 +1372,7 @@ class CephConfContext(object): return {} conf = config_flags_parser(conf) - if type(conf) != dict: + if not isinstance(conf, dict): log("Provided config-flags is not a dictionary - ignoring", level=WARNING) return {} diff --git a/tests/charmhelpers/fetch/__init__.py b/tests/charmhelpers/fetch/__init__.py index ec5e0fe9..480a6276 100644 --- a/tests/charmhelpers/fetch/__init__.py +++ b/tests/charmhelpers/fetch/__init__.py @@ -48,6 +48,13 @@ class AptLockError(Exception): pass +class GPGKeyError(Exception): + """Exception occurs when a GPG key cannot be fetched or used. The message + indicates what the problem is. + """ + pass + + class BaseFetchHandler(object): """Base class for FetchHandler implementations in fetch plugins""" @@ -77,21 +84,22 @@ module = "charmhelpers.fetch.%s" % __platform__ fetch = importlib.import_module(module) filter_installed_packages = fetch.filter_installed_packages -install = fetch.install -upgrade = fetch.upgrade -update = fetch.update -purge = fetch.purge +install = fetch.apt_install +upgrade = fetch.apt_upgrade +update = _fetch_update = fetch.apt_update +purge = fetch.apt_purge add_source = fetch.add_source if __platform__ == "ubuntu": apt_cache = fetch.apt_cache - apt_install = fetch.install - apt_update = fetch.update - apt_upgrade = fetch.upgrade - apt_purge = fetch.purge + apt_install = fetch.apt_install + apt_update = fetch.apt_update + apt_upgrade = fetch.apt_upgrade + apt_purge = fetch.apt_purge apt_mark = fetch.apt_mark apt_hold = fetch.apt_hold apt_unhold = fetch.apt_unhold + import_key = fetch.import_key get_upstream_version = fetch.get_upstream_version elif __platform__ == "centos": yum_search = fetch.yum_search @@ -135,7 +143,7 @@ def configure_sources(update=False, for source, key in zip(sources, keys): add_source(source, key) if update: - fetch.update(fatal=True) + _fetch_update(fatal=True) def install_remote(source, *args, **kwargs): diff --git a/tests/charmhelpers/fetch/centos.py b/tests/charmhelpers/fetch/centos.py index 604bbfb5..a91dcff0 100644 --- a/tests/charmhelpers/fetch/centos.py +++ b/tests/charmhelpers/fetch/centos.py @@ -132,7 +132,7 @@ def add_source(source, key=None): key_file.write(key) key_file.flush() key_file.seek(0) - subprocess.check_call(['rpm', '--import', key_file]) + subprocess.check_call(['rpm', '--import', key_file.name]) else: subprocess.check_call(['rpm', '--import', key]) diff --git a/tests/charmhelpers/fetch/snap.py b/tests/charmhelpers/fetch/snap.py index 23c707b0..112a54c3 100644 --- a/tests/charmhelpers/fetch/snap.py +++ b/tests/charmhelpers/fetch/snap.py @@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer: https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html """ import subprocess -from os import environ +import os from time import sleep from charmhelpers.core.hookenv import log __author__ = 'Joseph Borg ' -SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +# The return code for "couldn't acquire lock" in Snap +# (hopefully this will be improved). +SNAP_NO_LOCK = 1 SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +SNAP_CHANNELS = [ + 'edge', + 'beta', + 'candidate', + 'stable', +] class CouldNotAcquireLockException(Exception): @@ -47,13 +55,17 @@ def _snap_exec(commands): while return_code is None or return_code == SNAP_NO_LOCK: try: - return_code = subprocess.check_call(['snap'] + commands, env=environ) + return_code = subprocess.check_call(['snap'] + commands, + env=os.environ) except subprocess.CalledProcessError as e: retry_count += + 1 if retry_count > SNAP_NO_LOCK_RETRY_COUNT: - raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + raise CouldNotAcquireLockException( + 'Could not aquire lock after {} attempts' + .format(SNAP_NO_LOCK_RETRY_COUNT)) return_code = e.returncode - log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + log('Snap failed to acquire lock, trying again in {} seconds.' + .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN')) sleep(SNAP_NO_LOCK_RETRY_DELAY) return return_code diff --git a/tests/charmhelpers/fetch/ubuntu.py b/tests/charmhelpers/fetch/ubuntu.py index 7bc6cc7e..545348ff 100644 --- a/tests/charmhelpers/fetch/ubuntu.py +++ b/tests/charmhelpers/fetch/ubuntu.py @@ -12,29 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import OrderedDict import os +import platform +import re import six import time import subprocess - from tempfile import NamedTemporaryFile + from charmhelpers.core.host import ( lsb_release ) -from charmhelpers.core.hookenv import log -from charmhelpers.fetch import SourceConfigError +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) +from charmhelpers.fetch import SourceConfigError, GPGKeyError +PROPOSED_POCKET = ( + "# Proposed\n" + "deb http://archive.ubuntu.com/ubuntu {}-proposed main universe " + "multiverse restricted\n") +PROPOSED_PORTS_POCKET = ( + "# Proposed\n" + "deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe " + "multiverse restricted\n") +# Only supports 64bit and ppc64 at the moment. +ARCH_TO_PROPOSED_POCKET = { + 'x86_64': PROPOSED_POCKET, + 'ppc64le': PROPOSED_PORTS_POCKET, + 'aarch64': PROPOSED_PORTS_POCKET, +} +CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" +CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main """ - -PROPOSED_POCKET = """# Proposed -deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted -""" - CLOUD_ARCHIVE_POCKETS = { # Folsom 'folsom': 'precise-updates/folsom', + 'folsom/updates': 'precise-updates/folsom', 'precise-folsom': 'precise-updates/folsom', 'precise-folsom/updates': 'precise-updates/folsom', 'precise-updates/folsom': 'precise-updates/folsom', @@ -43,6 +61,7 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/folsom': 'precise-proposed/folsom', # Grizzly 'grizzly': 'precise-updates/grizzly', + 'grizzly/updates': 'precise-updates/grizzly', 'precise-grizzly': 'precise-updates/grizzly', 'precise-grizzly/updates': 'precise-updates/grizzly', 'precise-updates/grizzly': 'precise-updates/grizzly', @@ -51,6 +70,7 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/grizzly': 'precise-proposed/grizzly', # Havana 'havana': 'precise-updates/havana', + 'havana/updates': 'precise-updates/havana', 'precise-havana': 'precise-updates/havana', 'precise-havana/updates': 'precise-updates/havana', 'precise-updates/havana': 'precise-updates/havana', @@ -59,6 +79,7 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/havana': 'precise-proposed/havana', # Icehouse 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', 'precise-icehouse': 'precise-updates/icehouse', 'precise-icehouse/updates': 'precise-updates/icehouse', 'precise-updates/icehouse': 'precise-updates/icehouse', @@ -67,6 +88,7 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-proposed/icehouse': 'precise-proposed/icehouse', # Juno 'juno': 'trusty-updates/juno', + 'juno/updates': 'trusty-updates/juno', 'trusty-juno': 'trusty-updates/juno', 'trusty-juno/updates': 'trusty-updates/juno', 'trusty-updates/juno': 'trusty-updates/juno', @@ -75,6 +97,7 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-proposed/juno': 'trusty-proposed/juno', # Kilo 'kilo': 'trusty-updates/kilo', + 'kilo/updates': 'trusty-updates/kilo', 'trusty-kilo': 'trusty-updates/kilo', 'trusty-kilo/updates': 'trusty-updates/kilo', 'trusty-updates/kilo': 'trusty-updates/kilo', @@ -83,6 +106,7 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-proposed/kilo': 'trusty-proposed/kilo', # Liberty 'liberty': 'trusty-updates/liberty', + 'liberty/updates': 'trusty-updates/liberty', 'trusty-liberty': 'trusty-updates/liberty', 'trusty-liberty/updates': 'trusty-updates/liberty', 'trusty-updates/liberty': 'trusty-updates/liberty', @@ -91,6 +115,7 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-proposed/liberty': 'trusty-proposed/liberty', # Mitaka 'mitaka': 'trusty-updates/mitaka', + 'mitaka/updates': 'trusty-updates/mitaka', 'trusty-mitaka': 'trusty-updates/mitaka', 'trusty-mitaka/updates': 'trusty-updates/mitaka', 'trusty-updates/mitaka': 'trusty-updates/mitaka', @@ -99,6 +124,7 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', # Newton 'newton': 'xenial-updates/newton', + 'newton/updates': 'xenial-updates/newton', 'xenial-newton': 'xenial-updates/newton', 'xenial-newton/updates': 'xenial-updates/newton', 'xenial-updates/newton': 'xenial-updates/newton', @@ -107,12 +133,13 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-proposed/newton': 'xenial-proposed/newton', # Ocata 'ocata': 'xenial-updates/ocata', + 'ocata/updates': 'xenial-updates/ocata', 'xenial-ocata': 'xenial-updates/ocata', 'xenial-ocata/updates': 'xenial-updates/ocata', 'xenial-updates/ocata': 'xenial-updates/ocata', 'ocata/proposed': 'xenial-proposed/ocata', 'xenial-ocata/proposed': 'xenial-proposed/ocata', - 'xenial-ocata/newton': 'xenial-proposed/ocata', + 'xenial-proposed/ocata': 'xenial-proposed/ocata', # Pike 'pike': 'xenial-updates/pike', 'xenial-pike': 'xenial-updates/pike', @@ -120,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/pike': 'xenial-updates/pike', 'pike/proposed': 'xenial-proposed/pike', 'xenial-pike/proposed': 'xenial-proposed/pike', - 'xenial-pike/newton': 'xenial-proposed/pike', + 'xenial-proposed/pike': 'xenial-proposed/pike', # Queens 'queens': 'xenial-updates/queens', 'xenial-queens': 'xenial-updates/queens', @@ -128,12 +155,13 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/queens': 'xenial-updates/queens', 'queens/proposed': 'xenial-proposed/queens', 'xenial-queens/proposed': 'xenial-proposed/queens', - 'xenial-queens/newton': 'xenial-proposed/queens', + 'xenial-proposed/queens': 'xenial-proposed/queens', } + APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. +CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -161,7 +189,7 @@ def apt_cache(in_memory=True, progress=None): return apt_pkg.Cache(progress) -def install(packages, options=None, fatal=False): +def apt_install(packages, options=None, fatal=False): """Install one or more packages.""" if options is None: options = ['--option=Dpkg::Options::=--force-confold'] @@ -178,7 +206,7 @@ def install(packages, options=None, fatal=False): _run_apt_command(cmd, fatal) -def upgrade(options=None, fatal=False, dist=False): +def apt_upgrade(options=None, fatal=False, dist=False): """Upgrade all packages.""" if options is None: options = ['--option=Dpkg::Options::=--force-confold'] @@ -193,13 +221,13 @@ def upgrade(options=None, fatal=False, dist=False): _run_apt_command(cmd, fatal) -def update(fatal=False): +def apt_update(fatal=False): """Update local apt cache.""" cmd = ['apt-get', 'update'] _run_apt_command(cmd, fatal) -def purge(packages, fatal=False): +def apt_purge(packages, fatal=False): """Purge one or more packages.""" cmd = ['apt-get', '--assume-yes', 'purge'] if isinstance(packages, six.string_types): @@ -233,7 +261,45 @@ def apt_unhold(packages, fatal=False): return apt_mark(packages, 'unhold', fatal=fatal) -def add_source(source, key=None): +def import_key(keyid): + """Import a key in either ASCII Armor or Radix64 format. + + `keyid` is either the keyid to fetch from a PGP server, or + the key in ASCII armor foramt. + + :param keyid: String of key (or key id). + :raises: GPGKeyError if the key could not be imported + """ + key = keyid.strip() + if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and + key.endswith('-----END PGP PUBLIC KEY BLOCK-----')): + log("PGP key found (looks like ASCII Armor format)", level=DEBUG) + log("Importing ASCII Armor PGP key", level=DEBUG) + with NamedTemporaryFile() as keyfile: + with open(keyfile.name, 'w') as fd: + fd.write(key) + fd.write("\n") + cmd = ['apt-key', 'add', keyfile.name] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + error = "Error importing PGP key '{}'".format(key) + log(error) + raise GPGKeyError(error) + else: + log("PGP key found (looks like Radix64 format)", level=DEBUG) + log("Importing PGP key from keyserver", level=DEBUG) + cmd = ['apt-key', 'adv', '--keyserver', + 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + error = "Error importing PGP key '{}'".format(key) + log(error) + raise GPGKeyError(error) + + +def add_source(source, key=None, fail_invalid=False): """Add a package source to this system. @param source: a URL or sources.list entry, as supported by @@ -249,6 +315,33 @@ def add_source(source, key=None): such as 'cloud:icehouse' 'distro' may be used as a noop + Full list of source specifications supported by the function are: + + 'distro': A NOP; i.e. it has no effect. + 'proposed': the proposed deb spec [2] is wrtten to + /etc/apt/sources.list/proposed + 'distro-proposed': adds -proposed to the debs [2] + 'ppa:': add-apt-repository --yes + 'deb ': add-apt-repository --yes deb + 'http://....': add-apt-repository --yes http://... + 'cloud-archive:': add-apt-repository -yes cloud-archive: + 'cloud:[-staging]': specify a Cloud Archive pocket with + optional staging version. If staging is used then the staging PPA [2] + with be used. If staging is NOT used then the cloud archive [3] will be + added, and the 'ubuntu-cloud-keyring' package will be added for the + current distro. + + Otherwise the source is not recognised and this is logged to the juju log. + However, no error is raised, unless sys_error_on_exit is True. + + [1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main + where {} is replaced with the derived pocket name. + [2] deb http://archive.ubuntu.com/ubuntu {}-proposed \ + main universe multiverse restricted + where {} is replaced with the lsb_release codename (e.g. xenial) + [3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu + to /etc/apt/sources.list.d/cloud-archive-list + @param key: A key to be added to the system's APT keyring and used to verify the signatures on packages. Ideally, this should be an ASCII format GPG public key including the block headers. A GPG key @@ -256,51 +349,142 @@ def add_source(source, key=None): available to retrieve the actual public key from a public keyserver placing your Juju environment at risk. ppa and cloud archive keys are securely added automtically, so sould not be provided. + + @param fail_invalid: (boolean) if True, then the function raises a + SourceConfigError is there is no matching installation source. + + @raises SourceConfigError() if for cloud:, the is not a + valid pocket in CLOUD_ARCHIVE_POCKETS """ + _mapping = OrderedDict([ + (r"^distro$", lambda: None), # This is a NOP + (r"^(?:proposed|distro-proposed)$", _add_proposed), + (r"^cloud-archive:(.*)$", _add_apt_repository), + (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository), + (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), + (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), + (r"^cloud:(.*)$", _add_cloud_pocket), + (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), + ]) if source is None: - log('Source is not present. Skipping') - return - - if (source.startswith('ppa:') or - source.startswith('http') or - source.startswith('deb ') or - source.startswith('cloud-archive:')): - cmd = ['add-apt-repository', '--yes', source] - _run_with_retries(cmd) - elif source.startswith('cloud:'): - install(filter_installed_packages(['ubuntu-cloud-keyring']), - fatal=True) - pocket = source.split(':')[-1] - if pocket not in CLOUD_ARCHIVE_POCKETS: - raise SourceConfigError( - 'Unsupported cloud: source option %s' % - pocket) - actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] - with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: - apt.write(CLOUD_ARCHIVE.format(actual_pocket)) - elif source == 'proposed': - release = lsb_release()['DISTRIB_CODENAME'] - with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: - apt.write(PROPOSED_POCKET.format(release)) - elif source == 'distro': - pass + source = '' + for r, fn in six.iteritems(_mapping): + m = re.match(r, source) + if m: + # call the assoicated function with the captured groups + # raises SourceConfigError on error. + fn(*m.groups()) + if key: + try: + import_key(key) + except GPGKeyError as e: + raise SourceConfigError(str(e)) + break else: - log("Unknown source: {!r}".format(source)) + # nothing matched. log an error and maybe sys.exit + err = "Unknown source: {!r}".format(source) + log(err) + if fail_invalid: + raise SourceConfigError(err) - if key: - if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: - with NamedTemporaryFile('w+') as key_file: - key_file.write(key) - key_file.flush() - key_file.seek(0) - subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file) - else: - # Note that hkp: is in no way a secure protocol. Using a - # GPG key id is pointless from a security POV unless you - # absolutely trust your network and DNS. - subprocess.check_call(['apt-key', 'adv', '--keyserver', - 'hkp://keyserver.ubuntu.com:80', '--recv', - key]) + +def _add_proposed(): + """Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list + + Uses lsb_release()['DISTRIB_CODENAME'] to determine the correct staza for + the deb line. + + For intel architecutres PROPOSED_POCKET is used for the release, but for + other architectures PROPOSED_PORTS_POCKET is used for the release. + """ + release = lsb_release()['DISTRIB_CODENAME'] + arch = platform.machine() + if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET): + raise SourceConfigError("Arch {} not supported for (distro-)proposed" + .format(arch)) + with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: + apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release)) + + +def _add_apt_repository(spec): + """Add the spec using add_apt_repository + + :param spec: the parameter to pass to add_apt_repository + """ + _run_with_retries(['add-apt-repository', '--yes', spec]) + + +def _add_cloud_pocket(pocket): + """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list + + Note that this overwrites the existing file if there is one. + + This function also converts the simple pocket in to the actual pocket using + the CLOUD_ARCHIVE_POCKETS mapping. + + :param pocket: string representing the pocket to add a deb spec for. + :raises: SourceConfigError if the cloud pocket doesn't exist or the + requested release doesn't match the current distro version. + """ + apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), + fatal=True) + if pocket not in CLOUD_ARCHIVE_POCKETS: + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) + actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] + with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: + apt.write(CLOUD_ARCHIVE.format(actual_pocket)) + + +def _add_cloud_staging(cloud_archive_release, openstack_release): + """Add the cloud staging repository which is in + ppa:ubuntu-cloud-archive/-staging + + This function checks that the cloud_archive_release matches the current + codename for the distro that charm is being installed on. + + :param cloud_archive_release: string, codename for the release. + :param openstack_release: String, codename for the openstack release. + :raises: SourceConfigError if the cloud_archive_release doesn't match the + current version of the os. + """ + _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) + ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release) + cmd = 'add-apt-repository -y {}'.format(ppa) + _run_with_retries(cmd.split(' ')) + + +def _add_cloud_distro_check(cloud_archive_release, openstack_release): + """Add the cloud pocket, but also check the cloud_archive_release against + the current distro, and use the openstack_release as the full lookup. + + This just calls _add_cloud_pocket() with the openstack_release as pocket + to get the correct cloud-archive.list for dpkg to work with. + + :param cloud_archive_release:String, codename for the distro release. + :param openstack_release: String, spec for the release to look up in the + CLOUD_ARCHIVE_POCKETS + :raises: SourceConfigError if this is the wrong distro, or the pocket spec + doesn't exist. + """ + _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) + _add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release)) + + +def _verify_is_ubuntu_rel(release, os_release): + """Verify that the release is in the same as the current ubuntu release. + + :param release: String, lowercase for the release. + :param os_release: String, the os_release being asked for + :raises: SourceConfigError if the release is not the same as the ubuntu + release. + """ + ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] + if release != ubuntu_rel: + raise SourceConfigError( + 'Invalid Cloud Archive release specified: {}-{} on this Ubuntu' + 'version ({})'.format(release, os_release, ubuntu_rel)) def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), @@ -316,9 +500,12 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), :param: cmd_env: dict: Environment variables to add to the command run. """ - env = os.environ.copy() + env = None + kwargs = {} if cmd_env: + env = os.environ.copy() env.update(cmd_env) + kwargs['env'] = env if not retry_message: retry_message = "Failed executing '{}'".format(" ".join(cmd)) @@ -330,7 +517,8 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), retry_results = (None,) + retry_exitcodes while result in retry_results: try: - result = subprocess.check_call(cmd, env=env) + # result = subprocess.check_call(cmd, env=env) + result = subprocess.check_call(cmd, **kwargs) except subprocess.CalledProcessError as e: retry_count = retry_count + 1 if retry_count > max_retries: @@ -343,6 +531,7 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), def _run_apt_command(cmd, fatal=False): """Run an apt command with optional retries. + :param: cmd: str: The apt command to run. :param: fatal: bool: Whether the command's output should be checked and retried. """