openstack/charm-neutron-gateway
Code Issues Proposed changes

[corey.bryant,r=billy-olsen] Fix global requirements for git-deploy.

Browse Source
This commit is contained in:
Billy Olsen 2015-06-22 10:03:00 -07:00
commit ed9afce96c
13 changed files with 564 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hooks/charmhelpers/contrib/hahelpers/cluster.py
View File

@ -44,6 +44,7 @@ from charmhelpers.core.hookenv import (
ERROR,
WARNING,
unit_get,
is_leader as juju_is_leader
)
from charmhelpers.core.decorators import (
retry_on_exception,
@ -63,17 +64,30 @@ class CRMResourceNotFound(Exception):
pass
class CRMDCNotFound(Exception):
pass
def is_elected_leader(resource):
"""
Returns True if the charm executing this is the elected cluster leader.
It relies on two mechanisms to determine leadership:
1. If the charm is part of a corosync cluster, call corosync to
1. If juju is sufficiently new and leadership election is supported,
the is_leader command will be used.
2. If the charm is part of a corosync cluster, call corosync to
determine leadership.
2. If the charm is not part of a corosync cluster, the leader is
3. If the charm is not part of a corosync cluster, the leader is
determined as being "the alive unit with the lowest unit numer". In
other words, the oldest surviving unit.
"""
try:
return juju_is_leader()
except NotImplementedError:
log('Juju leadership election feature not enabled'
', using fallback support',
level=WARNING)
if is_clustered():
if not is_crm_leader(resource):
log('Deferring action to CRM leader.', level=INFO)
@ -106,8 +120,9 @@ def is_crm_dc():
status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
if not isinstance(status, six.text_type):
status = six.text_type(status, "utf-8")
except subprocess.CalledProcessError:
return False
except subprocess.CalledProcessError as ex:
raise CRMDCNotFound(str(ex))
current_dc = ''
for line in status.split('\n'):
if line.startswith('Current DC'):
@ -115,10 +130,14 @@ def is_crm_dc():
current_dc = line.split(':')[1].split()[0]
if current_dc == get_unit_hostname():
return True
elif current_dc == 'NONE':
raise CRMDCNotFound('Current DC: NONE')
return False
@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
@retry_on_exception(5, base_delay=2,
exc_type=(CRMResourceNotFound, CRMDCNotFound))
def is_crm_leader(resource, retry=False):
"""
Returns True if the charm calling this is the elected corosync leader,

8
hooks/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@ -110,7 +110,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
self.precise_havana, self.precise_icehouse,
self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
self.trusty_kilo, self.vivid_kilo) = range(10)
self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
self.wily_liberty) = range(12)
releases = {
('precise', None): self.precise_essex,
@ -121,8 +122,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('trusty', None): self.trusty_icehouse,
('trusty', 'cloud:trusty-juno'): self.trusty_juno,
('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
('utopic', None): self.utopic_juno,
('vivid', None): self.vivid_kilo}
('vivid', None): self.vivid_kilo,
('wily', None): self.wily_liberty}
return releases[(self.series, self.openstack)]
def _get_openstack_release_string(self):
@ -138,6 +141,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
('wily', 'liberty'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]

125
hooks/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -16,15 +16,15 @@
import logging
import os
import six
import time
import urllib
import glanceclient.v1.client as glance_client
import heatclient.v1.client as heat_client
import keystoneclient.v2_0 as keystone_client
import novaclient.v1_1.client as nova_client
import six
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
@ -37,7 +37,7 @@ class OpenStackAmuletUtils(AmuletUtils):
"""OpenStack amulet utilities.
This class inherits from AmuletUtils and has additional support
that is specifically for use by OpenStack charms.
that is specifically for use by OpenStack charm tests.
"""
def __init__(self, log_level=ERROR):
@ -51,6 +51,8 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate actual endpoint data vs expected endpoint data. The ports
are used to find the matching endpoint.
"""
self.log.debug('Validating endpoint data...')
self.log.debug('actual: {}'.format(repr(endpoints)))
found = False
for ep in endpoints:
self.log.debug('endpoint: {}'.format(repr(ep)))
@ -77,6 +79,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual service catalog endpoints vs a list of
expected service catalog endpoints.
"""
self.log.debug('Validating service catalog endpoint data...')
self.log.debug('actual: {}'.format(repr(actual)))
for k, v in six.iteritems(expected):
if k in actual:
@ -93,6 +96,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual tenant data vs list of expected tenant
data.
"""
self.log.debug('Validating tenant data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@ -114,6 +118,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual role data vs a list of expected role
data.
"""
self.log.debug('Validating role data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@ -134,6 +139,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual user data vs a list of expected user
data.
"""
self.log.debug('Validating user data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@ -155,17 +161,20 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual flavors vs a list of expected flavors.
"""
self.log.debug('Validating flavor data...')
self.log.debug('actual: {}'.format(repr(actual)))
act = [a.name for a in actual]
return self._validate_list_data(expected, act)
def tenant_exists(self, keystone, tenant):
"""Return True if tenant exists."""
self.log.debug('Checking if tenant exists ({})...'.format(tenant))
return tenant in [t.name for t in keystone.tenants.list()]
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant):
"""Authenticates admin user with the keystone admin endpoint."""
self.log.debug('Authenticating keystone admin...')
unit = keystone_sentry
service_ip = unit.relation('shared-db',
'mysql:shared-db')['private-address']
@ -175,6 +184,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_keystone_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with the keystone public endpoint."""
self.log.debug('Authenticating keystone user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return keystone_client.Client(username=user, password=password,
@ -182,12 +192,21 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_glance_admin(self, keystone):
"""Authenticates admin user with glance."""
self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
endpoint_type='adminURL')
return glance_client.Client(ep, token=keystone.auth_token)
def authenticate_heat_admin(self, keystone):
"""Authenticates the admin user with heat."""
self.log.debug('Authenticating heat admin...')
ep = keystone.service_catalog.url_for(service_type='orchestration',
endpoint_type='publicURL')
return heat_client.Client(endpoint=ep, token=keystone.auth_token)
def authenticate_nova_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with nova-api."""
self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return nova_client.Client(username=user, api_key=password,
@ -195,6 +214,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance."""
self.log.debug('Creating glance image ({})...'.format(image_name))
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@ -235,6 +255,11 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_image(self, glance, image):
"""Delete the specified image."""
# /!\ DEPRECATION WARNING
self.log.warn('/!\\ DEPRECATION WARNING: use '
'delete_resource instead of delete_image.')
self.log.debug('Deleting glance image ({})...'.format(image))
num_before = len(list(glance.images.list()))
glance.images.delete(image)
@ -254,6 +279,8 @@ class OpenStackAmuletUtils(AmuletUtils):
def create_instance(self, nova, image_name, instance_name, flavor):
"""Create the specified instance."""
self.log.debug('Creating instance '
'({}|{}|{})'.format(instance_name, image_name, flavor))
image = nova.images.find(name=image_name)
flavor = nova.flavors.find(name=flavor)
instance = nova.servers.create(name=instance_name, image=image,
@ -276,6 +303,11 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_instance(self, nova, instance):
"""Delete the specified instance."""
# /!\ DEPRECATION WARNING
self.log.warn('/!\\ DEPRECATION WARNING: use '
'delete_resource instead of delete_instance.')
self.log.debug('Deleting instance ({})...'.format(instance))
num_before = len(list(nova.servers.list()))
nova.servers.delete(instance)
@ -292,3 +324,90 @@ class OpenStackAmuletUtils(AmuletUtils):
return False
return True
def create_or_get_keypair(self, nova, keypair_name="testkey"):
"""Create a new keypair, or return pointer if it already exists."""
try:
_keypair = nova.keypairs.get(keypair_name)
self.log.debug('Keypair ({}) already exists, '
'using it.'.format(keypair_name))
return _keypair
except:
self.log.debug('Keypair ({}) does not exist, '
'creating it.'.format(keypair_name))
_keypair = nova.keypairs.create(name=keypair_name)
return _keypair
def delete_resource(self, resource, resource_id,
msg="resource", max_wait=120):
"""Delete one openstack resource, such as one instance, keypair,
image, volume, stack, etc., and confirm deletion within max wait time.
:param resource: pointer to os resource type, ex:glance_client.images
:param resource_id: unique name or id for the openstack resource
:param msg: text to identify purpose in logging
:param max_wait: maximum wait time in seconds
:returns: True if successful, otherwise False
"""
num_before = len(list(resource.list()))
resource.delete(resource_id)
tries = 0
num_after = len(list(resource.list()))
while num_after != (num_before - 1) and tries < (max_wait / 4):
self.log.debug('{} delete check: '
'{} [{}:{}] {}'.format(msg, tries,
num_before,
num_after,
resource_id))
time.sleep(4)
num_after = len(list(resource.list()))
tries += 1
self.log.debug('{}: expected, actual count = {}, '
'{}'.format(msg, num_before - 1, num_after))
if num_after == (num_before - 1):
return True
else:
self.log.error('{} delete timed out'.format(msg))
return False
def resource_reaches_status(self, resource, resource_id,
expected_stat='available',
msg='resource', max_wait=120):
"""Wait for an openstack resources status to reach an
expected status within a specified time. Useful to confirm that
nova instances, cinder vols, snapshots, glance images, heat stacks
and other resources eventually reach the expected status.
:param resource: pointer to os resource type, ex: heat_client.stacks
:param resource_id: unique id for the openstack resource
:param expected_stat: status to expect resource to reach
:param msg: text to identify purpose in logging
:param max_wait: maximum wait time in seconds
:returns: True if successful, False if status is not reached
"""
tries = 0
resource_stat = resource.get(resource_id).status
while resource_stat != expected_stat and tries < (max_wait / 4):
self.log.debug('{} status check: '
'{} [{}:{}] {}'.format(msg, tries,
resource_stat,
expected_stat,
resource_id))
time.sleep(4)
resource_stat = resource.get(resource_id).status
tries += 1
self.log.debug('{}: expected, actual status = {}, '
'{}'.format(msg, resource_stat, expected_stat))
if resource_stat == expected_stat:
return True
else:
self.log.debug('{} never reached expected status: '
'{}'.format(resource_id, expected_stat))
return False

2
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -240,7 +240,7 @@ class SharedDBContext(OSContextGenerator):
if self.relation_prefix:
password_setting = self.relation_prefix + '_password'
for rid in relation_ids('shared-db'):
for rid in relation_ids(self.interfaces[0]):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
host = rdata.get('db_host')

93
hooks/charmhelpers/contrib/openstack/ip.py
View File

@ -17,6 +17,7 @@
from charmhelpers.core.hookenv import (
config,
unit_get,
service_name,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
@ -26,8 +27,6 @@ from charmhelpers.contrib.network.ip import (
)
from charmhelpers.contrib.hahelpers.cluster import is_clustered
from functools import partial
PUBLIC = 'public'
INTERNAL = 'int'
ADMIN = 'admin'
@ -35,15 +34,18 @@ ADMIN = 'admin'
ADDRESS_MAP = {
PUBLIC: {
'config': 'os-public-network',
'fallback': 'public-address'
'fallback': 'public-address',
'override': 'os-public-hostname',
},
INTERNAL: {
'config': 'os-internal-network',
'fallback': 'private-address'
'fallback': 'private-address',
'override': 'os-internal-hostname',
},
ADMIN: {
'config': 'os-admin-network',
'fallback': 'private-address'
'fallback': 'private-address',
'override': 'os-admin-hostname',
}
}
@ -57,15 +59,50 @@ def canonical_url(configs, endpoint_type=PUBLIC):
:param endpoint_type: str endpoint type to resolve.
:param returns: str base URL for services on the current service unit.
"""
scheme = 'http'
if 'https' in configs.complete_contexts():
scheme = 'https'
scheme = _get_scheme(configs)
address = resolve_address(endpoint_type)
if is_ipv6(address):
address = "[{}]".format(address)
return '%s://%s' % (scheme, address)
def _get_scheme(configs):
"""Returns the scheme to use for the url (either http or https)
depending upon whether https is in the configs value.
:param configs: OSTemplateRenderer config templating object to inspect
for a complete https context.
:returns: either 'http' or 'https' depending on whether https is
configured within the configs context.
"""
scheme = 'http'
if configs and 'https' in configs.complete_contexts():
scheme = 'https'
return scheme
def _get_address_override(endpoint_type=PUBLIC):
"""Returns any address overrides that the user has defined based on the
endpoint type.
Note: this function allows for the service name to be inserted into the
address if the user specifies {service_name}.somehost.org.
:param endpoint_type: the type of endpoint to retrieve the override
value for.
:returns: any endpoint address or hostname that the user has overridden
or None if an override is not present.
"""
override_key = ADDRESS_MAP[endpoint_type]['override']
addr_override = config(override_key)
if not addr_override:
return None
else:
return addr_override.format(service_name=service_name())
def resolve_address(endpoint_type=PUBLIC):
"""Return unit address depending on net config.
@ -77,7 +114,10 @@ def resolve_address(endpoint_type=PUBLIC):
:param endpoint_type: Network endpoing type
"""
resolved_address = None
resolved_address = _get_address_override(endpoint_type)
if resolved_address:
return resolved_address
vips = config('vip')
if vips:
vips = vips.split()
@ -109,38 +149,3 @@ def resolve_address(endpoint_type=PUBLIC):
"clustered=%s)" % (net_type, clustered))
return resolved_address
def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC,
override=None):
"""Returns the correct endpoint URL to advertise to Keystone.
This method provides the correct endpoint URL which should be advertised to
the keystone charm for endpoint creation. This method allows for the url to
be overridden to force a keystone endpoint to have specific URL for any of
the defined scopes (admin, internal, public).
:param configs: OSTemplateRenderer config templating object to inspect
for a complete https context.
:param url_template: str format string for creating the url template. Only
two values will be passed - the scheme+hostname
returned by the canonical_url and the port.
:param endpoint_type: str endpoint type to resolve.
:param override: str the name of the config option which overrides the
endpoint URL defined by the charm itself. None will
disable any overrides (default).
"""
if override:
# Return any user-defined overrides for the keystone endpoint URL.
user_value = config(override)
if user_value:
return user_value.strip()
return url_template % (canonical_url(configs, endpoint_type), port)
public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC)
internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL)
admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN)

10
hooks/charmhelpers/contrib/openstack/neutron.py
View File

@ -172,14 +172,16 @@ def neutron_plugins():
'services': ['calico-felix',
'bird',
'neutron-dhcp-agent',
'nova-api-metadata'],
'nova-api-metadata',
'etcd'],
'packages': [[headers_package()] + determine_dkms_package(),
['calico-compute',
'bird',
'neutron-dhcp-agent',
'nova-api-metadata']],
'server_packages': ['neutron-server', 'calico-control'],
'server_services': ['neutron-server']
'nova-api-metadata',
'etcd']],
'server_packages': ['neutron-server', 'calico-control', 'etcd'],
'server_services': ['neutron-server', 'etcd']
},
'vsp': {
'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',

29
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -79,6 +79,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
('wily', 'liberty'),
])
@ -91,6 +92,7 @@ OPENSTACK_CODENAMES = OrderedDict([
('2014.1', 'icehouse'),
('2014.2', 'juno'),
('2015.1', 'kilo'),
('2015.2', 'liberty'),
])
# The ugly duckling
@ -113,6 +115,7 @@ SWIFT_CODENAMES = OrderedDict([
('2.2.0', 'juno'),
('2.2.1', 'kilo'),
('2.2.2', 'kilo'),
('2.3.0', 'liberty'),
])
DEFAULT_LOOPBACK_SIZE = '5G'
@ -321,6 +324,9 @@ def configure_installation_source(rel):
'kilo': 'trusty-updates/kilo',
'kilo/updates': 'trusty-updates/kilo',
'kilo/proposed': 'trusty-proposed/kilo',
'liberty': 'trusty-updates/liberty',
'liberty/updates': 'trusty-updates/liberty',
'liberty/proposed': 'trusty-proposed/liberty',
}
try:
@ -549,6 +555,11 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
# Upgrade setuptools from default virtualenv version. The default version
# in trusty breaks update.py in global requirements master branch.
pip_install('setuptools', upgrade=True, proxy=http_proxy,
venv=os.path.join(parent_dir, 'venv'))
for p in projects['repositories']:
repo = p['repository']
branch = p['branch']
@ -610,24 +621,24 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
else:
repo_dir = dest_dir
venv = os.path.join(parent_dir, 'venv')
if update_requirements:
if not requirements_dir:
error_out('requirements repo must be cloned before '
'updating from global requirements.')
_git_update_requirements(repo_dir, requirements_dir)
_git_update_requirements(venv, repo_dir, requirements_dir)
juju_log('Installing git repo from dir: {}'.format(repo_dir))
if http_proxy:
pip_install(repo_dir, proxy=http_proxy,
venv=os.path.join(parent_dir, 'venv'))
pip_install(repo_dir, proxy=http_proxy, venv=venv)
else:
pip_install(repo_dir,
venv=os.path.join(parent_dir, 'venv'))
pip_install(repo_dir, venv=venv)
return repo_dir
def _git_update_requirements(package_dir, reqs_dir):
def _git_update_requirements(venv, package_dir, reqs_dir):
"""
Update from global requirements.
@ -636,12 +647,14 @@ def _git_update_requirements(package_dir, reqs_dir):
"""
orig_dir = os.getcwd()
os.chdir(reqs_dir)
cmd = ['python', 'update.py', package_dir]
python = os.path.join(venv, 'bin/python')
cmd = [python, 'update.py', package_dir]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
package = os.path.basename(package_dir)
error_out("Error updating {} from global-requirements.txt".format(package))
error_out("Error updating {} from "
"global-requirements.txt".format(package))
os.chdir(orig_dir)

63
hooks/charmhelpers/core/hookenv.py
View File

@ -364,11 +364,16 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
relation_settings = relation_settings if relation_settings else {}
relation_cmd_line = ['relation-set']
accepts_file = "--file" in subprocess.check_output(
relation_cmd_line + ["--help"])
relation_cmd_line + ["--help"], universal_newlines=True)
if relation_id is not None:
relation_cmd_line.extend(('-r', relation_id))
settings = relation_settings.copy()
settings.update(kwargs)
for key, value in settings.items():
# Force value to be a string: it always should, but some call
# sites pass in things like dicts or numbers.
if value is not None:
settings[key] = "{}".format(value)
if accepts_file:
# --file was introduced in Juju 1.23.2. Use it by default if
# available, since otherwise we'll break if the relation data is
@ -390,6 +395,17 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
flush(local_unit())
def relation_clear(r_id=None):
''' Clears any relation data already set on relation r_id '''
settings = relation_get(rid=r_id,
unit=local_unit())
for setting in settings:
if setting not in ['public-address', 'private-address']:
settings[setting] = None
relation_set(relation_id=r_id,
**settings)
@cached
def relation_ids(reltype=None):
"""A list of relation_ids"""
@ -681,3 +697,48 @@ def status_get():
return 'unknown'
else:
raise
def translate_exc(from_exc, to_exc):
def inner_translate_exc1(f):
def inner_translate_exc2(*args, **kwargs):
try:
return f(*args, **kwargs)
except from_exc:
raise to_exc
return inner_translate_exc2
return inner_translate_exc1
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def is_leader():
"""Does the current unit hold the juju leadership
Uses juju to determine whether the current unit is the leader of its peers
"""
cmd = ['is-leader', '--format=json']
return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def leader_get(attribute=None):
"""Juju leader get value(s)"""
cmd = ['leader-get', '--format=json'] + [attribute or '-']
return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def leader_set(settings=None, **kwargs):
"""Juju leader set value(s)"""
log("Juju leader-set '%s'" % (settings), level=DEBUG)
cmd = ['leader-set']
settings = settings or {}
settings.update(kwargs)
for k, v in settings.iteritems():
if v is None:
cmd.append('{}='.format(k))
else:
cmd.append('{}={}'.format(k, v))
subprocess.check_call(cmd)

30
hooks/charmhelpers/core/host.py
View File

@ -24,6 +24,7 @@
import os
import re
import pwd
import glob
import grp
import random
import string
@ -269,6 +270,21 @@ def file_hash(path, hash_type='md5'):
return None
def path_hash(path):
"""
Generate a hash checksum of all files matching 'path'. Standard wildcards
like '*' and '?' are supported, see documentation for the 'glob' module for
more information.
:return: dict: A { filename: hash } dictionary for all matched files.
Empty if none found.
"""
return {
filename: file_hash(filename)
for filename in glob.iglob(path)
}
def check_hash(path, checksum, hash_type='md5'):
"""
Validate a file using a cryptographic checksum.
@ -296,23 +312,25 @@ def restart_on_change(restart_map, stopstart=False):
@restart_on_change({
'/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
'/etc/apache/sites-enabled/*': [ 'apache2' ]
})
def ceph_client_changed():
def config_changed():
pass # your code here
In this example, the cinder-api and cinder-volume services
would be restarted if /etc/ceph/ceph.conf is changed by the
ceph_client_changed function.
ceph_client_changed function. The apache2 service would be
restarted if any file matching the pattern got changed, created
or removed. Standard wildcards are supported, see documentation
for the 'glob' module for more information.
"""
def wrap(f):
def wrapped_f(*args, **kwargs):
checksums = {}
for path in restart_map:
checksums[path] = file_hash(path)
checksums = {path: path_hash(path) for path in restart_map}
f(*args, **kwargs)
restarts = []
for path in restart_map:
if checksums[path] != file_hash(path):
if path_hash(path) != checksums[path]:
restarts += restart_map[path]
services_list = list(OrderedDict.fromkeys(restarts))
if not stopstart:

39
hooks/charmhelpers/core/services/base.py
View File

@ -15,8 +15,8 @@
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
import os
import re
import json
from inspect import getargspec
from collections import Iterable, OrderedDict
from charmhelpers.core import host
@ -132,8 +132,8 @@ class ServiceManager(object):
if hook_name == 'stop':
self.stop_services()
else:
self.provide_data()
self.reconfigure_services()
self.provide_data()
cfg = hookenv.config()
if cfg.implicit_save:
cfg.save()
@ -145,15 +145,36 @@ class ServiceManager(object):
A provider must have a `name` attribute, which indicates which relation
to set data on, and a `provide_data()` method, which returns a dict of
data to set.
The `provide_data()` method can optionally accept two parameters:
* ``remote_service`` The name of the remote service that the data will
be provided to. The `provide_data()` method will be called once
for each connected service (not unit). This allows the method to
tailor its data to the given service.
* ``service_ready`` Whether or not the service definition had all of
its requirements met, and thus the ``data_ready`` callbacks run.
Note that the ``provided_data`` methods are now called **after** the
``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks
a chance to generate any data necessary for the providing to the remote
services.
"""
hook_name = hookenv.hook_name()
for service in self.services.values():
for service_name, service in self.services.items():
service_ready = self.is_ready(service_name)
for provider in service.get('provided_data', []):
if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
data = provider.provide_data()
_ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
if _ready:
hookenv.relation_set(None, data)
for relid in hookenv.relation_ids(provider.name):
units = hookenv.related_units(relid)
if not units:
continue
remote_service = units[0].split('/')[0]
argspec = getargspec(provider.provide_data)
if len(argspec.args) > 1:
data = provider.provide_data(remote_service, service_ready)
else:
data = provider.provide_data()
if data:
hookenv.relation_set(relid, data)
def reconfigure_services(self, *service_names):
"""

97
tests/charmhelpers/contrib/amulet/utils.py
View File

@ -15,13 +15,15 @@
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
import ConfigParser
import distro_info
import io
import logging
import os
import re
import six
import sys
import time
import six
import urlparse
class AmuletUtils(object):
@ -33,6 +35,7 @@ class AmuletUtils(object):
def __init__(self, log_level=logging.ERROR):
self.log = self.get_logger(level=log_level)
self.ubuntu_releases = self.get_ubuntu_releases()
def get_logger(self, name="amulet-logger", level=logging.DEBUG):
"""Get a logger object that will log to stdout."""
@ -70,12 +73,44 @@ class AmuletUtils(object):
else:
return False
def validate_services(self, commands):
"""Validate services.
def get_ubuntu_release_from_sentry(self, sentry_unit):
"""Get Ubuntu release codename from sentry unit.
Verify the specified services are running on the corresponding
:param sentry_unit: amulet sentry/service unit pointer
:returns: list of strings - release codename, failure message
"""
msg = None
cmd = 'lsb_release -cs'
release, code = sentry_unit.run(cmd)
if code == 0:
self.log.debug('{} lsb_release: {}'.format(
sentry_unit.info['unit_name'], release))
else:
msg = ('{} `{}` returned {} '
'{}'.format(sentry_unit.info['unit_name'],
cmd, release, code))
if release not in self.ubuntu_releases:
msg = ("Release ({}) not found in Ubuntu releases "
"({})".format(release, self.ubuntu_releases))
return release, msg
def validate_services(self, commands):
"""Validate that lists of commands succeed on service units. Can be
used to verify system services are running on the corresponding
service units.
"""
:param commands: dict with sentry keys and arbitrary command list vals
:returns: None if successful, Failure string message otherwise
"""
self.log.debug('Checking status of system services...')
# /!\ DEPRECATION WARNING (beisner):
# New and existing tests should be rewritten to use
# validate_services_by_name() as it is aware of init systems.
self.log.warn('/!\\ DEPRECATION WARNING: use '
'validate_services_by_name instead of validate_services '
'due to init system differences.')
for k, v in six.iteritems(commands):
for cmd in v:
output, code = k.run(cmd)
@ -86,6 +121,41 @@ class AmuletUtils(object):
return "command `{}` returned {}".format(cmd, str(code))
return None
def validate_services_by_name(self, sentry_services):
"""Validate system service status by service name, automatically
detecting init system based on Ubuntu release codename.
:param sentry_services: dict with sentry keys and svc list values
:returns: None if successful, Failure string message otherwise
"""
self.log.debug('Checking status of system services...')
# Point at which systemd became a thing
systemd_switch = self.ubuntu_releases.index('vivid')
for sentry_unit, services_list in six.iteritems(sentry_services):
# Get lsb_release codename from unit
release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
if ret:
return ret
for service_name in services_list:
if (self.ubuntu_releases.index(release) >= systemd_switch or
service_name == "rabbitmq-server"):
# init is systemd
cmd = 'sudo service {} status'.format(service_name)
elif self.ubuntu_releases.index(release) < systemd_switch:
# init is upstart
cmd = 'sudo status {}'.format(service_name)
output, code = sentry_unit.run(cmd)
self.log.debug('{} `{}` returned '
'{}'.format(sentry_unit.info['unit_name'],
cmd, code))
if code != 0:
return "command `{}` returned {}".format(cmd, str(code))
return None
def _get_config(self, unit, filename):
"""Get a ConfigParser object for parsing a unit's config file."""
file_contents = unit.file_contents(filename)
@ -104,6 +174,9 @@ class AmuletUtils(object):
Verify that the specified section of the config file contains
the expected option key:value pairs.
"""
self.log.debug('Validating config file data ({} in {} on {})'
'...'.format(section, config_file,
sentry_unit.info['unit_name']))
config = self._get_config(sentry_unit, config_file)
if section != 'DEFAULT' and not config.has_section(section):
@ -321,3 +394,15 @@ class AmuletUtils(object):
def endpoint_error(self, name, data):
return 'unexpected endpoint data in {} - {}'.format(name, data)
def get_ubuntu_releases(self):
"""Return a list of all Ubuntu releases in order of release."""
_d = distro_info.UbuntuDistroInfo()
_release_list = _d.all
self.log.debug('Ubuntu release list: {}'.format(_release_list))
return _release_list
def file_to_url(self, file_rel_path):
"""Convert a relative file path to a file URL."""
_abs_path = os.path.abspath(file_rel_path)
return urlparse.urlparse(_abs_path, scheme='file').geturl()

8
tests/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@ -110,7 +110,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
self.precise_havana, self.precise_icehouse,
self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
self.trusty_kilo, self.vivid_kilo) = range(10)
self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
self.wily_liberty) = range(12)
releases = {
('precise', None): self.precise_essex,
@ -121,8 +122,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('trusty', None): self.trusty_icehouse,
('trusty', 'cloud:trusty-juno'): self.trusty_juno,
('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
('utopic', None): self.utopic_juno,
('vivid', None): self.vivid_kilo}
('vivid', None): self.vivid_kilo,
('wily', None): self.wily_liberty}
return releases[(self.series, self.openstack)]
def _get_openstack_release_string(self):
@ -138,6 +141,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
('wily', 'liberty'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]

125
tests/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -16,15 +16,15 @@
import logging
import os
import six
import time
import urllib
import glanceclient.v1.client as glance_client
import heatclient.v1.client as heat_client
import keystoneclient.v2_0 as keystone_client
import novaclient.v1_1.client as nova_client
import six
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
@ -37,7 +37,7 @@ class OpenStackAmuletUtils(AmuletUtils):
"""OpenStack amulet utilities.
This class inherits from AmuletUtils and has additional support
that is specifically for use by OpenStack charms.
that is specifically for use by OpenStack charm tests.
"""
def __init__(self, log_level=ERROR):
@ -51,6 +51,8 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate actual endpoint data vs expected endpoint data. The ports
are used to find the matching endpoint.
"""
self.log.debug('Validating endpoint data...')
self.log.debug('actual: {}'.format(repr(endpoints)))
found = False
for ep in endpoints:
self.log.debug('endpoint: {}'.format(repr(ep)))
@ -77,6 +79,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual service catalog endpoints vs a list of
expected service catalog endpoints.
"""
self.log.debug('Validating service catalog endpoint data...')
self.log.debug('actual: {}'.format(repr(actual)))
for k, v in six.iteritems(expected):
if k in actual:
@ -93,6 +96,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual tenant data vs list of expected tenant
data.
"""
self.log.debug('Validating tenant data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@ -114,6 +118,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual role data vs a list of expected role
data.
"""
self.log.debug('Validating role data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@ -134,6 +139,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual user data vs a list of expected user
data.
"""
self.log.debug('Validating user data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@ -155,17 +161,20 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual flavors vs a list of expected flavors.
"""
self.log.debug('Validating flavor data...')
self.log.debug('actual: {}'.format(repr(actual)))
act = [a.name for a in actual]
return self._validate_list_data(expected, act)
def tenant_exists(self, keystone, tenant):
"""Return True if tenant exists."""
self.log.debug('Checking if tenant exists ({})...'.format(tenant))
return tenant in [t.name for t in keystone.tenants.list()]
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant):
"""Authenticates admin user with the keystone admin endpoint."""
self.log.debug('Authenticating keystone admin...')
unit = keystone_sentry
service_ip = unit.relation('shared-db',
'mysql:shared-db')['private-address']
@ -175,6 +184,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_keystone_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with the keystone public endpoint."""
self.log.debug('Authenticating keystone user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return keystone_client.Client(username=user, password=password,
@ -182,12 +192,21 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_glance_admin(self, keystone):
"""Authenticates admin user with glance."""
self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
endpoint_type='adminURL')
return glance_client.Client(ep, token=keystone.auth_token)
def authenticate_heat_admin(self, keystone):
"""Authenticates the admin user with heat."""
self.log.debug('Authenticating heat admin...')
ep = keystone.service_catalog.url_for(service_type='orchestration',
endpoint_type='publicURL')
return heat_client.Client(endpoint=ep, token=keystone.auth_token)
def authenticate_nova_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with nova-api."""
self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return nova_client.Client(username=user, api_key=password,
@ -195,6 +214,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance."""
self.log.debug('Creating glance image ({})...'.format(image_name))
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@ -235,6 +255,11 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_image(self, glance, image):
"""Delete the specified image."""
# /!\ DEPRECATION WARNING
self.log.warn('/!\\ DEPRECATION WARNING: use '
'delete_resource instead of delete_image.')
self.log.debug('Deleting glance image ({})...'.format(image))
num_before = len(list(glance.images.list()))
glance.images.delete(image)
@ -254,6 +279,8 @@ class OpenStackAmuletUtils(AmuletUtils):
def create_instance(self, nova, image_name, instance_name, flavor):
"""Create the specified instance."""
self.log.debug('Creating instance '
'({}|{}|{})'.format(instance_name, image_name, flavor))
image = nova.images.find(name=image_name)
flavor = nova.flavors.find(name=flavor)
instance = nova.servers.create(name=instance_name, image=image,
@ -276,6 +303,11 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_instance(self, nova, instance):
"""Delete the specified instance."""
# /!\ DEPRECATION WARNING
self.log.warn('/!\\ DEPRECATION WARNING: use '
'delete_resource instead of delete_instance.')
self.log.debug('Deleting instance ({})...'.format(instance))
num_before = len(list(nova.servers.list()))
nova.servers.delete(instance)
@ -292,3 +324,90 @@ class OpenStackAmuletUtils(AmuletUtils):
return False
return True
def create_or_get_keypair(self, nova, keypair_name="testkey"):
"""Create a new keypair, or return pointer if it already exists."""
try:
_keypair = nova.keypairs.get(keypair_name)
self.log.debug('Keypair ({}) already exists, '
'using it.'.format(keypair_name))
return _keypair
except:
self.log.debug('Keypair ({}) does not exist, '
'creating it.'.format(keypair_name))
_keypair = nova.keypairs.create(name=keypair_name)
return _keypair
def delete_resource(self, resource, resource_id,
msg="resource", max_wait=120):
"""Delete one openstack resource, such as one instance, keypair,
image, volume, stack, etc., and confirm deletion within max wait time.
:param resource: pointer to os resource type, ex:glance_client.images
:param resource_id: unique name or id for the openstack resource
:param msg: text to identify purpose in logging
:param max_wait: maximum wait time in seconds
:returns: True if successful, otherwise False
"""
num_before = len(list(resource.list()))
resource.delete(resource_id)
tries = 0
num_after = len(list(resource.list()))
while num_after != (num_before - 1) and tries < (max_wait / 4):
self.log.debug('{} delete check: '
'{} [{}:{}] {}'.format(msg, tries,
num_before,
num_after,
resource_id))
time.sleep(4)
num_after = len(list(resource.list()))
tries += 1
self.log.debug('{}: expected, actual count = {}, '
'{}'.format(msg, num_before - 1, num_after))
if num_after == (num_before - 1):
return True
else:
self.log.error('{} delete timed out'.format(msg))
return False
def resource_reaches_status(self, resource, resource_id,
expected_stat='available',
msg='resource', max_wait=120):
"""Wait for an openstack resources status to reach an
expected status within a specified time. Useful to confirm that
nova instances, cinder vols, snapshots, glance images, heat stacks
and other resources eventually reach the expected status.
:param resource: pointer to os resource type, ex: heat_client.stacks
:param resource_id: unique id for the openstack resource
:param expected_stat: status to expect resource to reach
:param msg: text to identify purpose in logging
:param max_wait: maximum wait time in seconds
:returns: True if successful, False if status is not reached
"""
tries = 0
resource_stat = resource.get(resource_id).status
while resource_stat != expected_stat and tries < (max_wait / 4):
self.log.debug('{} status check: '
'{} [{}:{}] {}'.format(msg, tries,
resource_stat,
expected_stat,
resource_id))
time.sleep(4)
resource_stat = resource.get(resource_id).status
tries += 1
self.log.debug('{}: expected, actual status = {}, '
'{}'.format(msg, resource_stat, expected_stat))
if resource_stat == expected_stat:
return True
else:
self.log.debug('{} never reached expected status: '
'{}'.format(resource_id, expected_stat))
return False