openstack/charm-nova-compute
Code Issues Proposed changes

Resync charm helpers

Browse Source 
Add support for OpenStack Newton and Ocata.

Rework version detection code to just match on major version for
OpenStack projects using semantic versioning.

Provide fallback version detection based on major.minor versions
for swift packages.

Rework config-flags support helpers.

Fix is_ip function to correctly detect both IPv4 and IPv6 addresses.

Change-Id: Ic03fee106099286e478ef7aa0723f5465d7ac7d1
This commit is contained in:
James Page 2016-05-27 11:16:53 +01:00
parent a5f8c755fe
commit fd3687eafb
6 changed files with 247 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hooks/charmhelpers/contrib/network/ip.py
View File

@ -405,10 +405,10 @@ def is_ip(address):
Returns True if address is a valid IP address.
"""
try:
# Test to see if already an IPv4 address
socket.inet_aton(address)
# Test to see if already an IPv4/IPv6 address
address = netaddr.IPAddress(address)
return True
except socket.error:
except netaddr.AddrFormatError:
return False

88
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -23,7 +23,6 @@ from base64 import b64decode
from subprocess import check_call, CalledProcessError
import six
import yaml
from charmhelpers.fetch import (
apt_install,
@ -50,6 +49,7 @@ from charmhelpers.core.hookenv import (
from charmhelpers.core.sysctl import create as sysctl_create
from charmhelpers.core.strutils import bool_from_string
from charmhelpers.contrib.openstack.exceptions import OSContextError
from charmhelpers.core.host import (
get_bond_master,
@ -88,7 +88,10 @@ from charmhelpers.contrib.network.ip import (
is_address_in_network,
is_bridge_member,
)
from charmhelpers.contrib.openstack.utils import get_host_ip
from charmhelpers.contrib.openstack.utils import (
config_flags_parser,
get_host_ip,
)
from charmhelpers.core.unitdata import kv
try:
@ -101,10 +104,6 @@ CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
ADDRESS_TYPES = ['admin', 'internal', 'public']
class OSContextError(Exception):
pass
def ensure_packages(packages):
"""Install but do not upgrade required plugin packages."""
required = filter_installed_packages(packages)
@ -125,83 +124,6 @@ def context_complete(ctxt):
return True
def config_flags_parser(config_flags):
"""Parses config flags string into dict.
This parsing method supports a few different formats for the config
flag values to be parsed:
1. A string in the simple format of key=value pairs, with the possibility
of specifying multiple key value pairs within the same string. For
example, a string in the format of 'key1=value1, key2=value2' will
return a dict of:
{'key1': 'value1',
'key2': 'value2'}.
2. A string in the above format, but supporting a comma-delimited list
of values for the same key. For example, a string in the format of
'key1=value1, key2=value3,value4,value5' will return a dict of:
{'key1', 'value1',
'key2', 'value2,value3,value4'}
3. A string containing a colon character (:) prior to an equal
character (=) will be treated as yaml and parsed as such. This can be
used to specify more complex key value pairs. For example,
a string in the format of 'key1: subkey1=value1, subkey2=value2' will
return a dict of:
{'key1', 'subkey1=value1, subkey2=value2'}
The provided config_flags string may be a list of comma-separated values
which themselves may be comma-separated list of values.
"""
# If we find a colon before an equals sign then treat it as yaml.
# Note: limit it to finding the colon first since this indicates assignment
# for inline yaml.
colon = config_flags.find(':')
equals = config_flags.find('=')
if colon > 0:
if colon < equals or equals < 0:
return yaml.safe_load(config_flags)
if config_flags.find('==') >= 0:
log("config_flags is not in expected format (key=value)", level=ERROR)
raise OSContextError
# strip the following from each value.
post_strippers = ' ,'
# we strip any leading/trailing '=' or ' ' from the string then
# split on '='.
split = config_flags.strip(' =').split('=')
limit = len(split)
flags = {}
for i in range(0, limit - 1):
current = split[i]
next = split[i + 1]
vindex = next.rfind(',')
if (i == limit - 2) or (vindex < 0):
value = next
else:
value = next[:vindex]
if i == 0:
key = current
else:
# if this not the first entry, expect an embedded key.
index = current.rfind(',')
if index < 0:
log("Invalid config value(s) at index %s" % (i), level=ERROR)
raise OSContextError
key = current[index + 1:]
# Add to collection.
flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
return flags
class OSContextGenerator(object):
"""Base class for all context generators."""
interfaces = []

6
hooks/charmhelpers/contrib/openstack/exceptions.py Normal file
View File

@ -0,0 +1,6 @@
class OSContextError(Exception):
"""Raised when an error occurs during context generation.
This exception is principally used in contrib.openstack.context
"""
pass

206
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -25,6 +25,7 @@ import sys
import re
import itertools
import functools
import shutil
import six
import tempfile
@ -46,6 +47,7 @@ from charmhelpers.core.hookenv import (
charm_dir,
DEBUG,
INFO,
ERROR,
related_units,
relation_ids,
relation_set,
@ -82,6 +84,7 @@ from charmhelpers.core.host import (
from charmhelpers.fetch import apt_install, apt_cache, install_remote
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
from charmhelpers.contrib.openstack.exceptions import OSContextError
CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
@ -100,6 +103,8 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('vivid', 'kilo'),
('wily', 'liberty'),
('xenial', 'mitaka'),
('yakkety', 'newton'),
('zebra', 'ocata'), # TODO: upload with real Z name
])
@ -114,6 +119,8 @@ OPENSTACK_CODENAMES = OrderedDict([
('2015.1', 'kilo'),
('2015.2', 'liberty'),
('2016.1', 'mitaka'),
('2016.2', 'newton'),
('2017.1', 'ocata'),
])
# The ugly duckling - must list releases oldest to newest
@ -138,47 +145,65 @@ SWIFT_CODENAMES = OrderedDict([
['2.3.0', '2.4.0', '2.5.0']),
('mitaka',
['2.5.0', '2.6.0', '2.7.0']),
('newton',
['2.8.0']),
])
# >= Liberty version->codename mapping
PACKAGE_CODENAMES = {
'nova-common': OrderedDict([
('12.0', 'liberty'),
('13.0', 'mitaka'),
('12', 'liberty'),
('13', 'mitaka'),
('14', 'newton'),
('15', 'ocata'),
]),
'neutron-common': OrderedDict([
('7.0', 'liberty'),
('8.0', 'mitaka'),
('8.1', 'mitaka'),
('7', 'liberty'),
('8', 'mitaka'),
('9', 'newton'),
('10', 'ocata'),
]),
'cinder-common': OrderedDict([
('7.0', 'liberty'),
('8.0', 'mitaka'),
('7', 'liberty'),
('8', 'mitaka'),
('9', 'newton'),
('10', 'ocata'),
]),
'keystone': OrderedDict([
('8.0', 'liberty'),
('8.1', 'liberty'),
('9.0', 'mitaka'),
('8', 'liberty'),
('9', 'mitaka'),
('10', 'newton'),
('11', 'ocata'),
]),
'horizon-common': OrderedDict([
('8.0', 'liberty'),
('9.0', 'mitaka'),
('8', 'liberty'),
('9', 'mitaka'),
('10', 'newton'),
('11', 'ocata'),
]),
'ceilometer-common': OrderedDict([
('5.0', 'liberty'),
('6.0', 'mitaka'),
('5', 'liberty'),
('6', 'mitaka'),
('7', 'newton'),
('8', 'ocata'),
]),
'heat-common': OrderedDict([
('5.0', 'liberty'),
('6.0', 'mitaka'),
('5', 'liberty'),
('6', 'mitaka'),
('7', 'newton'),
('8', 'ocata'),
]),
'glance-common': OrderedDict([
('11.0', 'liberty'),
('12.0', 'mitaka'),
('11', 'liberty'),
('12', 'mitaka'),
('13', 'newton'),
('14', 'ocata'),
]),
'openstack-dashboard': OrderedDict([
('8.0', 'liberty'),
('9.0', 'mitaka'),
('8', 'liberty'),
('9', 'mitaka'),
('10', 'newton'),
('11', 'ocata'),
]),
}
@ -254,6 +279,7 @@ def get_os_version_codename_swift(codename):
def get_swift_codename(version):
'''Determine OpenStack codename that corresponds to swift version.'''
codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v]
if len(codenames) > 1:
# If more than one release codename contains this version we determine
# the actual codename based on the highest available install source.
@ -265,6 +291,16 @@ def get_swift_codename(version):
return codename
elif len(codenames) == 1:
return codenames[0]
# NOTE: fallback - attempt to match with just major.minor version
match = re.match('^(\d+)\.(\d+)', version)
if match:
major_minor_version = match.group(0)
for codename, versions in six.iteritems(SWIFT_CODENAMES):
for release_version in versions:
if release_version.startswith(major_minor_version):
return codename
return None
@ -303,10 +339,13 @@ def get_os_codename_package(package, fatal=True):
if match:
vers = match.group(0)
# Generate a major version number for newer semantic
# versions of openstack projects
major_vers = vers.split('.')[0]
# >= Liberty independent project versions
if (package in PACKAGE_CODENAMES and
vers in PACKAGE_CODENAMES[package]):
return PACKAGE_CODENAMES[package][vers]
major_vers in PACKAGE_CODENAMES[package]):
return PACKAGE_CODENAMES[package][major_vers]
else:
# < Liberty co-ordinated project versions
try:
@ -466,6 +505,9 @@ def configure_installation_source(rel):
'mitaka': 'trusty-updates/mitaka',
'mitaka/updates': 'trusty-updates/mitaka',
'mitaka/proposed': 'trusty-proposed/mitaka',
'newton': 'xenial-updates/newton',
'newton/updates': 'xenial-updates/newton',
'newton/proposed': 'xenial-proposed/newton',
}
try:
@ -858,6 +900,47 @@ def git_yaml_value(projects_yaml, key):
return None
def git_generate_systemd_init_files(templates_dir):
"""
Generate systemd init files.
Generates and installs systemd init units and script files based on the
*.init.in files contained in the templates_dir directory.
This code is based on the openstack-pkg-tools package and its init
script generation, which is used by the OpenStack packages.
"""
for f in os.listdir(templates_dir):
if f.endswith(".init.in"):
init_in_file = f
init_file = f[:-8]
service_file = "{}.service".format(init_file)
init_in_source = os.path.join(templates_dir, init_in_file)
init_source = os.path.join(templates_dir, init_file)
service_source = os.path.join(templates_dir, service_file)
init_dest = os.path.join('/etc/init.d', init_file)
service_dest = os.path.join('/lib/systemd/system', service_file)
shutil.copyfile(init_in_source, init_source)
with open(init_source, 'a') as outfile:
template = '/usr/share/openstack-pkg-tools/init-script-template'
with open(template) as infile:
outfile.write('\n\n{}'.format(infile.read()))
cmd = ['pkgos-gen-systemd-unit', init_in_source]
subprocess.check_call(cmd)
if os.path.exists(init_dest):
os.remove(init_dest)
if os.path.exists(service_dest):
os.remove(service_dest)
shutil.move(init_source, init_dest)
shutil.move(service_source, service_dest)
os.chmod(init_dest, 0o755)
def os_workload_status(configs, required_interfaces, charm_func=None):
"""
Decorator to set workload status based on complete contexts
@ -1574,3 +1657,82 @@ def pausable_restart_on_change(restart_map, stopstart=False,
restart_functions)
return wrapped_f
return wrap
def config_flags_parser(config_flags):
"""Parses config flags string into dict.
This parsing method supports a few different formats for the config
flag values to be parsed:
1. A string in the simple format of key=value pairs, with the possibility
of specifying multiple key value pairs within the same string. For
example, a string in the format of 'key1=value1, key2=value2' will
return a dict of:
{'key1': 'value1',
'key2': 'value2'}.
2. A string in the above format, but supporting a comma-delimited list
of values for the same key. For example, a string in the format of
'key1=value1, key2=value3,value4,value5' will return a dict of:
{'key1', 'value1',
'key2', 'value2,value3,value4'}
3. A string containing a colon character (:) prior to an equal
character (=) will be treated as yaml and parsed as such. This can be
used to specify more complex key value pairs. For example,
a string in the format of 'key1: subkey1=value1, subkey2=value2' will
return a dict of:
{'key1', 'subkey1=value1, subkey2=value2'}
The provided config_flags string may be a list of comma-separated values
which themselves may be comma-separated list of values.
"""
# If we find a colon before an equals sign then treat it as yaml.
# Note: limit it to finding the colon first since this indicates assignment
# for inline yaml.
colon = config_flags.find(':')
equals = config_flags.find('=')
if colon > 0:
if colon < equals or equals < 0:
return yaml.safe_load(config_flags)
if config_flags.find('==') >= 0:
juju_log("config_flags is not in expected format (key=value)",
level=ERROR)
raise OSContextError
# strip the following from each value.
post_strippers = ' ,'
# we strip any leading/trailing '=' or ' ' from the string then
# split on '='.
split = config_flags.strip(' =').split('=')
limit = len(split)
flags = {}
for i in range(0, limit - 1):
current = split[i]
next = split[i + 1]
vindex = next.rfind(',')
if (i == limit - 2) or (vindex < 0):
value = next
else:
value = next[:vindex]
if i == 0:
key = current
else:
# if this not the first entry, expect an embedded key.
index = current.rfind(',')
if index < 0:
juju_log("Invalid config value(s) at index %s" % (i),
level=ERROR)
raise OSContextError
key = current[index + 1:]
# Add to collection.
flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
return flags

41
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -40,6 +40,7 @@ from subprocess import (
CalledProcessError,
)
from charmhelpers.core.hookenv import (
config,
local_unit,
relation_get,
relation_ids,
@ -64,6 +65,7 @@ from charmhelpers.fetch import (
)
from charmhelpers.core.kernel import modprobe
from charmhelpers.contrib.openstack.utils import config_flags_parser
KEYRING = '/etc/ceph/ceph.client.{}.keyring'
KEYFILE = '/etc/ceph/ceph.client.{}.key'
@ -1204,3 +1206,42 @@ def send_request_if_needed(request, relation='ceph'):
for rid in relation_ids(relation):
log('Sending request {}'.format(request.request_id), level=DEBUG)
relation_set(relation_id=rid, broker_req=request.request)
class CephConfContext(object):
"""Ceph config (ceph.conf) context.
Supports user-provided Ceph configuration settings. Use can provide a
dictionary as the value for the config-flags charm option containing
Ceph configuration settings keyede by their section in ceph.conf.
"""
def __init__(self, permitted_sections=None):
self.permitted_sections = permitted_sections or []
def __call__(self):
conf = config('config-flags')
if not conf:
return {}
conf = config_flags_parser(conf)
if type(conf) != dict:
log("Provided config-flags is not a dictionary - ignoring",
level=WARNING)
return {}
permitted = self.permitted_sections
if permitted:
diff = set(conf.keys()).symmetric_difference(set(permitted))
if diff:
log("Config-flags contains invalid keys '%s' - they will be "
"ignored" % (', '.join(diff)), level=WARNING)
ceph_conf = {}
for key in conf:
if permitted and key not in permitted:
log("Ignoring key '%s'" % key, level=WARNING)
continue
ceph_conf[key] = conf[key]
return ceph_conf

8
hooks/charmhelpers/fetch/__init__.py
View File

@ -106,6 +106,14 @@ CLOUD_ARCHIVE_POCKETS = {
'mitaka/proposed': 'trusty-proposed/mitaka',
'trusty-mitaka/proposed': 'trusty-proposed/mitaka',
'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
# Newton
'newton': 'xenial-updates/newton',
'xenial-newton': 'xenial-updates/newton',
'xenial-newton/updates': 'xenial-updates/newton',
'xenial-updates/newton': 'xenial-updates/newton',
'newton/proposed': 'xenial-proposed/newton',
'xenial-newton/proposed': 'xenial-proposed/newton',
'xenial-proposed/newton': 'xenial-proposed/newton',
}
# The order of this list is very important. Handlers should be listed in from