From fd3687eafb41768eb3b1cf928a80460cb3f080d1 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 27 May 2016 11:16:53 +0100 Subject: [PATCH] Resync charm helpers Add support for OpenStack Newton and Ocata. Rework version detection code to just match on major version for OpenStack projects using semantic versioning. Provide fallback version detection based on major.minor versions for swift packages. Rework config-flags support helpers. Fix is_ip function to correctly detect both IPv4 and IPv6 addresses. Change-Id: Ic03fee106099286e478ef7aa0723f5465d7ac7d1 --- hooks/charmhelpers/contrib/network/ip.py | 6 +- .../charmhelpers/contrib/openstack/context.py | 88 +------- .../contrib/openstack/exceptions.py | 6 + hooks/charmhelpers/contrib/openstack/utils.py | 206 ++++++++++++++++-- .../contrib/storage/linux/ceph.py | 41 ++++ hooks/charmhelpers/fetch/__init__.py | 8 + 6 files changed, 247 insertions(+), 108 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/exceptions.py diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 6bba07b6..99d78f2f 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -405,10 +405,10 @@ def is_ip(address): Returns True if address is a valid IP address. """ try: - # Test to see if already an IPv4 address - socket.inet_aton(address) + # Test to see if already an IPv4/IPv6 address + address = netaddr.IPAddress(address) return True - except socket.error: + except netaddr.AddrFormatError: return False diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c07b33dd..5faa7eda 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -23,7 +23,6 @@ from base64 import b64decode from subprocess import check_call, CalledProcessError import six -import yaml from charmhelpers.fetch import ( apt_install, @@ -50,6 +49,7 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.sysctl import create as sysctl_create from charmhelpers.core.strutils import bool_from_string +from charmhelpers.contrib.openstack.exceptions import OSContextError from charmhelpers.core.host import ( get_bond_master, @@ -88,7 +88,10 @@ from charmhelpers.contrib.network.ip import ( is_address_in_network, is_bridge_member, ) -from charmhelpers.contrib.openstack.utils import get_host_ip +from charmhelpers.contrib.openstack.utils import ( + config_flags_parser, + get_host_ip, +) from charmhelpers.core.unitdata import kv try: @@ -101,10 +104,6 @@ CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] -class OSContextError(Exception): - pass - - def ensure_packages(packages): """Install but do not upgrade required plugin packages.""" required = filter_installed_packages(packages) @@ -125,83 +124,6 @@ def context_complete(ctxt): return True -def config_flags_parser(config_flags): - """Parses config flags string into dict. - - This parsing method supports a few different formats for the config - flag values to be parsed: - - 1. A string in the simple format of key=value pairs, with the possibility - of specifying multiple key value pairs within the same string. For - example, a string in the format of 'key1=value1, key2=value2' will - return a dict of: - - {'key1': 'value1', - 'key2': 'value2'}. - - 2. A string in the above format, but supporting a comma-delimited list - of values for the same key. For example, a string in the format of - 'key1=value1, key2=value3,value4,value5' will return a dict of: - - {'key1', 'value1', - 'key2', 'value2,value3,value4'} - - 3. A string containing a colon character (:) prior to an equal - character (=) will be treated as yaml and parsed as such. This can be - used to specify more complex key value pairs. For example, - a string in the format of 'key1: subkey1=value1, subkey2=value2' will - return a dict of: - - {'key1', 'subkey1=value1, subkey2=value2'} - - The provided config_flags string may be a list of comma-separated values - which themselves may be comma-separated list of values. - """ - # If we find a colon before an equals sign then treat it as yaml. - # Note: limit it to finding the colon first since this indicates assignment - # for inline yaml. - colon = config_flags.find(':') - equals = config_flags.find('=') - if colon > 0: - if colon < equals or equals < 0: - return yaml.safe_load(config_flags) - - if config_flags.find('==') >= 0: - log("config_flags is not in expected format (key=value)", level=ERROR) - raise OSContextError - - # strip the following from each value. - post_strippers = ' ,' - # we strip any leading/trailing '=' or ' ' from the string then - # split on '='. - split = config_flags.strip(' =').split('=') - limit = len(split) - flags = {} - for i in range(0, limit - 1): - current = split[i] - next = split[i + 1] - vindex = next.rfind(',') - if (i == limit - 2) or (vindex < 0): - value = next - else: - value = next[:vindex] - - if i == 0: - key = current - else: - # if this not the first entry, expect an embedded key. - index = current.rfind(',') - if index < 0: - log("Invalid config value(s) at index %s" % (i), level=ERROR) - raise OSContextError - key = current[index + 1:] - - # Add to collection. - flags[key.strip(post_strippers)] = value.rstrip(post_strippers) - - return flags - - class OSContextGenerator(object): """Base class for all context generators.""" interfaces = [] diff --git a/hooks/charmhelpers/contrib/openstack/exceptions.py b/hooks/charmhelpers/contrib/openstack/exceptions.py new file mode 100644 index 00000000..ea4eb68e --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/exceptions.py @@ -0,0 +1,6 @@ +class OSContextError(Exception): + """Raised when an error occurs during context generation. + + This exception is principally used in contrib.openstack.context + """ + pass diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index e64a106c..bd6efc48 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -25,6 +25,7 @@ import sys import re import itertools import functools +import shutil import six import tempfile @@ -46,6 +47,7 @@ from charmhelpers.core.hookenv import ( charm_dir, DEBUG, INFO, + ERROR, related_units, relation_ids, relation_set, @@ -82,6 +84,7 @@ from charmhelpers.core.host import ( from charmhelpers.fetch import apt_install, apt_cache, install_remote from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device +from charmhelpers.contrib.openstack.exceptions import OSContextError CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' @@ -100,6 +103,8 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('vivid', 'kilo'), ('wily', 'liberty'), ('xenial', 'mitaka'), + ('yakkety', 'newton'), + ('zebra', 'ocata'), # TODO: upload with real Z name ]) @@ -114,6 +119,8 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2015.1', 'kilo'), ('2015.2', 'liberty'), ('2016.1', 'mitaka'), + ('2016.2', 'newton'), + ('2017.1', 'ocata'), ]) # The ugly duckling - must list releases oldest to newest @@ -138,47 +145,65 @@ SWIFT_CODENAMES = OrderedDict([ ['2.3.0', '2.4.0', '2.5.0']), ('mitaka', ['2.5.0', '2.6.0', '2.7.0']), + ('newton', + ['2.8.0']), ]) # >= Liberty version->codename mapping PACKAGE_CODENAMES = { 'nova-common': OrderedDict([ - ('12.0', 'liberty'), - ('13.0', 'mitaka'), + ('12', 'liberty'), + ('13', 'mitaka'), + ('14', 'newton'), + ('15', 'ocata'), ]), 'neutron-common': OrderedDict([ - ('7.0', 'liberty'), - ('8.0', 'mitaka'), - ('8.1', 'mitaka'), + ('7', 'liberty'), + ('8', 'mitaka'), + ('9', 'newton'), + ('10', 'ocata'), ]), 'cinder-common': OrderedDict([ - ('7.0', 'liberty'), - ('8.0', 'mitaka'), + ('7', 'liberty'), + ('8', 'mitaka'), + ('9', 'newton'), + ('10', 'ocata'), ]), 'keystone': OrderedDict([ - ('8.0', 'liberty'), - ('8.1', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), 'horizon-common': OrderedDict([ - ('8.0', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), 'ceilometer-common': OrderedDict([ - ('5.0', 'liberty'), - ('6.0', 'mitaka'), + ('5', 'liberty'), + ('6', 'mitaka'), + ('7', 'newton'), + ('8', 'ocata'), ]), 'heat-common': OrderedDict([ - ('5.0', 'liberty'), - ('6.0', 'mitaka'), + ('5', 'liberty'), + ('6', 'mitaka'), + ('7', 'newton'), + ('8', 'ocata'), ]), 'glance-common': OrderedDict([ - ('11.0', 'liberty'), - ('12.0', 'mitaka'), + ('11', 'liberty'), + ('12', 'mitaka'), + ('13', 'newton'), + ('14', 'ocata'), ]), 'openstack-dashboard': OrderedDict([ - ('8.0', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), } @@ -254,6 +279,7 @@ def get_os_version_codename_swift(codename): def get_swift_codename(version): '''Determine OpenStack codename that corresponds to swift version.''' codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] + if len(codenames) > 1: # If more than one release codename contains this version we determine # the actual codename based on the highest available install source. @@ -265,6 +291,16 @@ def get_swift_codename(version): return codename elif len(codenames) == 1: return codenames[0] + + # NOTE: fallback - attempt to match with just major.minor version + match = re.match('^(\d+)\.(\d+)', version) + if match: + major_minor_version = match.group(0) + for codename, versions in six.iteritems(SWIFT_CODENAMES): + for release_version in versions: + if release_version.startswith(major_minor_version): + return codename + return None @@ -303,10 +339,13 @@ def get_os_codename_package(package, fatal=True): if match: vers = match.group(0) + # Generate a major version number for newer semantic + # versions of openstack projects + major_vers = vers.split('.')[0] # >= Liberty independent project versions if (package in PACKAGE_CODENAMES and - vers in PACKAGE_CODENAMES[package]): - return PACKAGE_CODENAMES[package][vers] + major_vers in PACKAGE_CODENAMES[package]): + return PACKAGE_CODENAMES[package][major_vers] else: # < Liberty co-ordinated project versions try: @@ -466,6 +505,9 @@ def configure_installation_source(rel): 'mitaka': 'trusty-updates/mitaka', 'mitaka/updates': 'trusty-updates/mitaka', 'mitaka/proposed': 'trusty-proposed/mitaka', + 'newton': 'xenial-updates/newton', + 'newton/updates': 'xenial-updates/newton', + 'newton/proposed': 'xenial-proposed/newton', } try: @@ -858,6 +900,47 @@ def git_yaml_value(projects_yaml, key): return None +def git_generate_systemd_init_files(templates_dir): + """ + Generate systemd init files. + + Generates and installs systemd init units and script files based on the + *.init.in files contained in the templates_dir directory. + + This code is based on the openstack-pkg-tools package and its init + script generation, which is used by the OpenStack packages. + """ + for f in os.listdir(templates_dir): + if f.endswith(".init.in"): + init_in_file = f + init_file = f[:-8] + service_file = "{}.service".format(init_file) + + init_in_source = os.path.join(templates_dir, init_in_file) + init_source = os.path.join(templates_dir, init_file) + service_source = os.path.join(templates_dir, service_file) + + init_dest = os.path.join('/etc/init.d', init_file) + service_dest = os.path.join('/lib/systemd/system', service_file) + + shutil.copyfile(init_in_source, init_source) + with open(init_source, 'a') as outfile: + template = '/usr/share/openstack-pkg-tools/init-script-template' + with open(template) as infile: + outfile.write('\n\n{}'.format(infile.read())) + + cmd = ['pkgos-gen-systemd-unit', init_in_source] + subprocess.check_call(cmd) + + if os.path.exists(init_dest): + os.remove(init_dest) + if os.path.exists(service_dest): + os.remove(service_dest) + shutil.move(init_source, init_dest) + shutil.move(service_source, service_dest) + os.chmod(init_dest, 0o755) + + def os_workload_status(configs, required_interfaces, charm_func=None): """ Decorator to set workload status based on complete contexts @@ -1574,3 +1657,82 @@ def pausable_restart_on_change(restart_map, stopstart=False, restart_functions) return wrapped_f return wrap + + +def config_flags_parser(config_flags): + """Parses config flags string into dict. + + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + + {'key1', 'subkey1=value1, subkey2=value2'} + + The provided config_flags string may be a list of comma-separated values + which themselves may be comma-separated list of values. + """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + + if config_flags.find('==') >= 0: + juju_log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) + flags = {} + for i in range(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + juju_log("Invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + + return flags diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index d008081f..2528f5cf 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -40,6 +40,7 @@ from subprocess import ( CalledProcessError, ) from charmhelpers.core.hookenv import ( + config, local_unit, relation_get, relation_ids, @@ -64,6 +65,7 @@ from charmhelpers.fetch import ( ) from charmhelpers.core.kernel import modprobe +from charmhelpers.contrib.openstack.utils import config_flags_parser KEYRING = '/etc/ceph/ceph.client.{}.keyring' KEYFILE = '/etc/ceph/ceph.client.{}.key' @@ -1204,3 +1206,42 @@ def send_request_if_needed(request, relation='ceph'): for rid in relation_ids(relation): log('Sending request {}'.format(request.request_id), level=DEBUG) relation_set(relation_id=rid, broker_req=request.request) + + +class CephConfContext(object): + """Ceph config (ceph.conf) context. + + Supports user-provided Ceph configuration settings. Use can provide a + dictionary as the value for the config-flags charm option containing + Ceph configuration settings keyede by their section in ceph.conf. + """ + def __init__(self, permitted_sections=None): + self.permitted_sections = permitted_sections or [] + + def __call__(self): + conf = config('config-flags') + if not conf: + return {} + + conf = config_flags_parser(conf) + if type(conf) != dict: + log("Provided config-flags is not a dictionary - ignoring", + level=WARNING) + return {} + + permitted = self.permitted_sections + if permitted: + diff = set(conf.keys()).symmetric_difference(set(permitted)) + if diff: + log("Config-flags contains invalid keys '%s' - they will be " + "ignored" % (', '.join(diff)), level=WARNING) + + ceph_conf = {} + for key in conf: + if permitted and key not in permitted: + log("Ignoring key '%s'" % key, level=WARNING) + continue + + ceph_conf[key] = conf[key] + + return ceph_conf diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index db0d86a2..ad485ec8 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -106,6 +106,14 @@ CLOUD_ARCHIVE_POCKETS = { 'mitaka/proposed': 'trusty-proposed/mitaka', 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', + # Newton + 'newton': 'xenial-updates/newton', + 'xenial-newton': 'xenial-updates/newton', + 'xenial-newton/updates': 'xenial-updates/newton', + 'xenial-updates/newton': 'xenial-updates/newton', + 'newton/proposed': 'xenial-proposed/newton', + 'xenial-newton/proposed': 'xenial-proposed/newton', + 'xenial-proposed/newton': 'xenial-proposed/newton', } # The order of this list is very important. Handlers should be listed in from