diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 6bba07b..99d78f2 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -405,10 +405,10 @@ def is_ip(address): Returns True if address is a valid IP address. """ try: - # Test to see if already an IPv4 address - socket.inet_aton(address) + # Test to see if already an IPv4/IPv6 address + address = netaddr.IPAddress(address) return True - except socket.error: + except netaddr.AddrFormatError: return False diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c07b33d..5faa7ed 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -23,7 +23,6 @@ from base64 import b64decode from subprocess import check_call, CalledProcessError import six -import yaml from charmhelpers.fetch import ( apt_install, @@ -50,6 +49,7 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.sysctl import create as sysctl_create from charmhelpers.core.strutils import bool_from_string +from charmhelpers.contrib.openstack.exceptions import OSContextError from charmhelpers.core.host import ( get_bond_master, @@ -88,7 +88,10 @@ from charmhelpers.contrib.network.ip import ( is_address_in_network, is_bridge_member, ) -from charmhelpers.contrib.openstack.utils import get_host_ip +from charmhelpers.contrib.openstack.utils import ( + config_flags_parser, + get_host_ip, +) from charmhelpers.core.unitdata import kv try: @@ -101,10 +104,6 @@ CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] -class OSContextError(Exception): - pass - - def ensure_packages(packages): """Install but do not upgrade required plugin packages.""" required = filter_installed_packages(packages) @@ -125,83 +124,6 @@ def context_complete(ctxt): return True -def config_flags_parser(config_flags): - """Parses config flags string into dict. - - This parsing method supports a few different formats for the config - flag values to be parsed: - - 1. A string in the simple format of key=value pairs, with the possibility - of specifying multiple key value pairs within the same string. For - example, a string in the format of 'key1=value1, key2=value2' will - return a dict of: - - {'key1': 'value1', - 'key2': 'value2'}. - - 2. A string in the above format, but supporting a comma-delimited list - of values for the same key. For example, a string in the format of - 'key1=value1, key2=value3,value4,value5' will return a dict of: - - {'key1', 'value1', - 'key2', 'value2,value3,value4'} - - 3. A string containing a colon character (:) prior to an equal - character (=) will be treated as yaml and parsed as such. This can be - used to specify more complex key value pairs. For example, - a string in the format of 'key1: subkey1=value1, subkey2=value2' will - return a dict of: - - {'key1', 'subkey1=value1, subkey2=value2'} - - The provided config_flags string may be a list of comma-separated values - which themselves may be comma-separated list of values. - """ - # If we find a colon before an equals sign then treat it as yaml. - # Note: limit it to finding the colon first since this indicates assignment - # for inline yaml. - colon = config_flags.find(':') - equals = config_flags.find('=') - if colon > 0: - if colon < equals or equals < 0: - return yaml.safe_load(config_flags) - - if config_flags.find('==') >= 0: - log("config_flags is not in expected format (key=value)", level=ERROR) - raise OSContextError - - # strip the following from each value. - post_strippers = ' ,' - # we strip any leading/trailing '=' or ' ' from the string then - # split on '='. - split = config_flags.strip(' =').split('=') - limit = len(split) - flags = {} - for i in range(0, limit - 1): - current = split[i] - next = split[i + 1] - vindex = next.rfind(',') - if (i == limit - 2) or (vindex < 0): - value = next - else: - value = next[:vindex] - - if i == 0: - key = current - else: - # if this not the first entry, expect an embedded key. - index = current.rfind(',') - if index < 0: - log("Invalid config value(s) at index %s" % (i), level=ERROR) - raise OSContextError - key = current[index + 1:] - - # Add to collection. - flags[key.strip(post_strippers)] = value.rstrip(post_strippers) - - return flags - - class OSContextGenerator(object): """Base class for all context generators.""" interfaces = [] diff --git a/hooks/charmhelpers/contrib/openstack/exceptions.py b/hooks/charmhelpers/contrib/openstack/exceptions.py new file mode 100644 index 0000000..ea4eb68 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/exceptions.py @@ -0,0 +1,6 @@ +class OSContextError(Exception): + """Raised when an error occurs during context generation. + + This exception is principally used in contrib.openstack.context + """ + pass diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index e64a106..bd6efc4 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -25,6 +25,7 @@ import sys import re import itertools import functools +import shutil import six import tempfile @@ -46,6 +47,7 @@ from charmhelpers.core.hookenv import ( charm_dir, DEBUG, INFO, + ERROR, related_units, relation_ids, relation_set, @@ -82,6 +84,7 @@ from charmhelpers.core.host import ( from charmhelpers.fetch import apt_install, apt_cache, install_remote from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device +from charmhelpers.contrib.openstack.exceptions import OSContextError CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' @@ -100,6 +103,8 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('vivid', 'kilo'), ('wily', 'liberty'), ('xenial', 'mitaka'), + ('yakkety', 'newton'), + ('zebra', 'ocata'), # TODO: upload with real Z name ]) @@ -114,6 +119,8 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2015.1', 'kilo'), ('2015.2', 'liberty'), ('2016.1', 'mitaka'), + ('2016.2', 'newton'), + ('2017.1', 'ocata'), ]) # The ugly duckling - must list releases oldest to newest @@ -138,47 +145,65 @@ SWIFT_CODENAMES = OrderedDict([ ['2.3.0', '2.4.0', '2.5.0']), ('mitaka', ['2.5.0', '2.6.0', '2.7.0']), + ('newton', + ['2.8.0']), ]) # >= Liberty version->codename mapping PACKAGE_CODENAMES = { 'nova-common': OrderedDict([ - ('12.0', 'liberty'), - ('13.0', 'mitaka'), + ('12', 'liberty'), + ('13', 'mitaka'), + ('14', 'newton'), + ('15', 'ocata'), ]), 'neutron-common': OrderedDict([ - ('7.0', 'liberty'), - ('8.0', 'mitaka'), - ('8.1', 'mitaka'), + ('7', 'liberty'), + ('8', 'mitaka'), + ('9', 'newton'), + ('10', 'ocata'), ]), 'cinder-common': OrderedDict([ - ('7.0', 'liberty'), - ('8.0', 'mitaka'), + ('7', 'liberty'), + ('8', 'mitaka'), + ('9', 'newton'), + ('10', 'ocata'), ]), 'keystone': OrderedDict([ - ('8.0', 'liberty'), - ('8.1', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), 'horizon-common': OrderedDict([ - ('8.0', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), 'ceilometer-common': OrderedDict([ - ('5.0', 'liberty'), - ('6.0', 'mitaka'), + ('5', 'liberty'), + ('6', 'mitaka'), + ('7', 'newton'), + ('8', 'ocata'), ]), 'heat-common': OrderedDict([ - ('5.0', 'liberty'), - ('6.0', 'mitaka'), + ('5', 'liberty'), + ('6', 'mitaka'), + ('7', 'newton'), + ('8', 'ocata'), ]), 'glance-common': OrderedDict([ - ('11.0', 'liberty'), - ('12.0', 'mitaka'), + ('11', 'liberty'), + ('12', 'mitaka'), + ('13', 'newton'), + ('14', 'ocata'), ]), 'openstack-dashboard': OrderedDict([ - ('8.0', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), } @@ -254,6 +279,7 @@ def get_os_version_codename_swift(codename): def get_swift_codename(version): '''Determine OpenStack codename that corresponds to swift version.''' codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] + if len(codenames) > 1: # If more than one release codename contains this version we determine # the actual codename based on the highest available install source. @@ -265,6 +291,16 @@ def get_swift_codename(version): return codename elif len(codenames) == 1: return codenames[0] + + # NOTE: fallback - attempt to match with just major.minor version + match = re.match('^(\d+)\.(\d+)', version) + if match: + major_minor_version = match.group(0) + for codename, versions in six.iteritems(SWIFT_CODENAMES): + for release_version in versions: + if release_version.startswith(major_minor_version): + return codename + return None @@ -303,10 +339,13 @@ def get_os_codename_package(package, fatal=True): if match: vers = match.group(0) + # Generate a major version number for newer semantic + # versions of openstack projects + major_vers = vers.split('.')[0] # >= Liberty independent project versions if (package in PACKAGE_CODENAMES and - vers in PACKAGE_CODENAMES[package]): - return PACKAGE_CODENAMES[package][vers] + major_vers in PACKAGE_CODENAMES[package]): + return PACKAGE_CODENAMES[package][major_vers] else: # < Liberty co-ordinated project versions try: @@ -466,6 +505,9 @@ def configure_installation_source(rel): 'mitaka': 'trusty-updates/mitaka', 'mitaka/updates': 'trusty-updates/mitaka', 'mitaka/proposed': 'trusty-proposed/mitaka', + 'newton': 'xenial-updates/newton', + 'newton/updates': 'xenial-updates/newton', + 'newton/proposed': 'xenial-proposed/newton', } try: @@ -858,6 +900,47 @@ def git_yaml_value(projects_yaml, key): return None +def git_generate_systemd_init_files(templates_dir): + """ + Generate systemd init files. + + Generates and installs systemd init units and script files based on the + *.init.in files contained in the templates_dir directory. + + This code is based on the openstack-pkg-tools package and its init + script generation, which is used by the OpenStack packages. + """ + for f in os.listdir(templates_dir): + if f.endswith(".init.in"): + init_in_file = f + init_file = f[:-8] + service_file = "{}.service".format(init_file) + + init_in_source = os.path.join(templates_dir, init_in_file) + init_source = os.path.join(templates_dir, init_file) + service_source = os.path.join(templates_dir, service_file) + + init_dest = os.path.join('/etc/init.d', init_file) + service_dest = os.path.join('/lib/systemd/system', service_file) + + shutil.copyfile(init_in_source, init_source) + with open(init_source, 'a') as outfile: + template = '/usr/share/openstack-pkg-tools/init-script-template' + with open(template) as infile: + outfile.write('\n\n{}'.format(infile.read())) + + cmd = ['pkgos-gen-systemd-unit', init_in_source] + subprocess.check_call(cmd) + + if os.path.exists(init_dest): + os.remove(init_dest) + if os.path.exists(service_dest): + os.remove(service_dest) + shutil.move(init_source, init_dest) + shutil.move(service_source, service_dest) + os.chmod(init_dest, 0o755) + + def os_workload_status(configs, required_interfaces, charm_func=None): """ Decorator to set workload status based on complete contexts @@ -1574,3 +1657,82 @@ def pausable_restart_on_change(restart_map, stopstart=False, restart_functions) return wrapped_f return wrap + + +def config_flags_parser(config_flags): + """Parses config flags string into dict. + + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + + {'key1', 'subkey1=value1, subkey2=value2'} + + The provided config_flags string may be a list of comma-separated values + which themselves may be comma-separated list of values. + """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + + if config_flags.find('==') >= 0: + juju_log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) + flags = {} + for i in range(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + juju_log("Invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + + return flags diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index d008081..2528f5c 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -40,6 +40,7 @@ from subprocess import ( CalledProcessError, ) from charmhelpers.core.hookenv import ( + config, local_unit, relation_get, relation_ids, @@ -64,6 +65,7 @@ from charmhelpers.fetch import ( ) from charmhelpers.core.kernel import modprobe +from charmhelpers.contrib.openstack.utils import config_flags_parser KEYRING = '/etc/ceph/ceph.client.{}.keyring' KEYFILE = '/etc/ceph/ceph.client.{}.key' @@ -1204,3 +1206,42 @@ def send_request_if_needed(request, relation='ceph'): for rid in relation_ids(relation): log('Sending request {}'.format(request.request_id), level=DEBUG) relation_set(relation_id=rid, broker_req=request.request) + + +class CephConfContext(object): + """Ceph config (ceph.conf) context. + + Supports user-provided Ceph configuration settings. Use can provide a + dictionary as the value for the config-flags charm option containing + Ceph configuration settings keyede by their section in ceph.conf. + """ + def __init__(self, permitted_sections=None): + self.permitted_sections = permitted_sections or [] + + def __call__(self): + conf = config('config-flags') + if not conf: + return {} + + conf = config_flags_parser(conf) + if type(conf) != dict: + log("Provided config-flags is not a dictionary - ignoring", + level=WARNING) + return {} + + permitted = self.permitted_sections + if permitted: + diff = set(conf.keys()).symmetric_difference(set(permitted)) + if diff: + log("Config-flags contains invalid keys '%s' - they will be " + "ignored" % (', '.join(diff)), level=WARNING) + + ceph_conf = {} + for key in conf: + if permitted and key not in permitted: + log("Ignoring key '%s'" % key, level=WARNING) + continue + + ceph_conf[key] = conf[key] + + return ceph_conf diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index db0d86a..ad485ec 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -106,6 +106,14 @@ CLOUD_ARCHIVE_POCKETS = { 'mitaka/proposed': 'trusty-proposed/mitaka', 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', + # Newton + 'newton': 'xenial-updates/newton', + 'xenial-newton': 'xenial-updates/newton', + 'xenial-newton/updates': 'xenial-updates/newton', + 'xenial-updates/newton': 'xenial-updates/newton', + 'newton/proposed': 'xenial-proposed/newton', + 'xenial-newton/proposed': 'xenial-proposed/newton', + 'xenial-proposed/newton': 'xenial-proposed/newton', } # The order of this list is very important. Handlers should be listed in from