openstack/charm-swift-storage
Code Issues Proposed changes

[coreycb,r=james-page] Fixup amulet tests.

Browse Source
This commit is contained in:
James Page 2014-10-06 22:43:12 +01:00
commit ea5c67d3c7
11 changed files with 186 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -15,7 +15,8 @@ test:
# coreycb note: The -v should only be temporary until Amulet sends
# raise_status() messages to stderr:
# https://bugs.launchpad.net/amulet/+bug/1320357
@juju test -v -p AMULET_HTTP_PROXY
@juju test -v -p AMULET_HTTP_PROXY --timeout 900 \
00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse
bin/charm_helpers_sync.py:
@mkdir -p bin

2
hooks/charmhelpers/contrib/network/ip.py
View File

@ -57,6 +57,8 @@ def get_address_in_network(network, fallback=None, fatal=False):
else:
if fatal:
not_found_error_out()
else:
return None
_validate_cidr(network)
network = netaddr.IPNetwork(network)

61
hooks/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@ -1,6 +1,3 @@
from bzrlib.branch import Branch
import os
import re
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
@ -13,62 +10,62 @@ class OpenStackAmuletDeployment(AmuletDeployment):
that is specifically for use by OpenStack charms.
"""
def __init__(self, series=None, openstack=None, source=None):
def __init__(self, series=None, openstack=None, source=None, stable=True):
"""Initialize the deployment environment."""
super(OpenStackAmuletDeployment, self).__init__(series)
self.openstack = openstack
self.source = source
def _is_dev_branch(self):
"""Determine if branch being tested is a dev (i.e. next) branch."""
branch = Branch.open(os.getcwd())
parent = branch.get_parent()
pattern = re.compile("^.*/next/$")
if (pattern.match(parent)):
return True
else:
return False
self.stable = stable
# Note(coreycb): this needs to be changed when new next branches come
# out.
self.current_next = "trusty"
def _determine_branch_locations(self, other_services):
"""Determine the branch locations for the other services.
If the branch being tested is a dev branch, then determine the
development branch locations for the other services. Otherwise,
the default charm store branches will be used."""
name = 0
if self._is_dev_branch():
updated_services = []
Determine if the local branch being tested is derived from its
stable or next (dev) branch, and based on this, use the corresonding
stable or next branches for the other_services."""
base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
if self.stable:
for svc in other_services:
if svc[name] in ['mysql', 'mongodb', 'rabbitmq-server']:
location = 'lp:charms/{}'.format(svc[name])
temp = 'lp:charms/{}'
svc['location'] = temp.format(svc['name'])
else:
for svc in other_services:
if svc['name'] in base_charms:
temp = 'lp:charms/{}'
svc['location'] = temp.format(svc['name'])
else:
temp = 'lp:~openstack-charmers/charms/trusty/{}/next'
location = temp.format(svc[name])
updated_services.append(svc + (location,))
other_services = updated_services
temp = 'lp:~openstack-charmers/charms/{}/{}/next'
svc['location'] = temp.format(self.current_next,
svc['name'])
return other_services
def _add_services(self, this_service, other_services):
"""Add services to the deployment and set openstack-origin/source."""
name = 0
other_services = self._determine_branch_locations(other_services)
super(OpenStackAmuletDeployment, self)._add_services(this_service,
other_services)
services = other_services
services.append(this_service)
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph']
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
'ceph-osd', 'ceph-radosgw']
if self.openstack:
for svc in services:
if svc[name] not in use_source:
if svc['name'] not in use_source:
config = {'openstack-origin': self.openstack}
self.d.configure(svc[name], config)
self.d.configure(svc['name'], config)
if self.source:
for svc in services:
if svc[name] in use_source:
if svc['name'] in use_source:
config = {'source': self.source}
self.d.configure(svc[name], config)
self.d.configure(svc['name'], config)
def _configure_services(self, configs):
"""Configure all of the services."""

92
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -52,6 +52,7 @@ from charmhelpers.contrib.openstack.neutron import (
from charmhelpers.contrib.network.ip import (
get_address_in_network,
get_ipv6_addr,
get_netmask_for_address,
format_ipv6_addr,
is_address_in_network
)
@ -408,6 +409,9 @@ class CephContext(OSContextGenerator):
return ctxt
ADDRESS_TYPES = ['admin', 'internal', 'public']
class HAProxyContext(OSContextGenerator):
interfaces = ['cluster']
@ -420,7 +424,6 @@ class HAProxyContext(OSContextGenerator):
if not relation_ids('cluster'):
return {}
cluster_hosts = {}
l_unit = local_unit().replace('/', '-')
if config('prefer-ipv6'):
@ -428,17 +431,49 @@ class HAProxyContext(OSContextGenerator):
else:
addr = unit_get('private-address')
cluster_hosts[l_unit] = get_address_in_network(config('os-internal-network'),
addr)
cluster_hosts = {}
for rid in relation_ids('cluster'):
for unit in related_units(rid):
_unit = unit.replace('/', '-')
addr = relation_get('private-address', rid=rid, unit=unit)
cluster_hosts[_unit] = addr
# NOTE(jamespage): build out map of configured network endpoints
# and associated backends
for addr_type in ADDRESS_TYPES:
laddr = get_address_in_network(
config('os-{}-network'.format(addr_type)))
if laddr:
cluster_hosts[laddr] = {}
cluster_hosts[laddr]['network'] = "{}/{}".format(
laddr,
get_netmask_for_address(laddr)
)
cluster_hosts[laddr]['backends'] = {}
cluster_hosts[laddr]['backends'][l_unit] = laddr
for rid in relation_ids('cluster'):
for unit in related_units(rid):
_unit = unit.replace('/', '-')
_laddr = relation_get('{}-address'.format(addr_type),
rid=rid, unit=unit)
if _laddr:
cluster_hosts[laddr]['backends'][_unit] = _laddr
# NOTE(jamespage) no split configurations found, just use
# private addresses
if not cluster_hosts:
cluster_hosts[addr] = {}
cluster_hosts[addr]['network'] = "{}/{}".format(
addr,
get_netmask_for_address(addr)
)
cluster_hosts[addr]['backends'] = {}
cluster_hosts[addr]['backends'][l_unit] = addr
for rid in relation_ids('cluster'):
for unit in related_units(rid):
_unit = unit.replace('/', '-')
_laddr = relation_get('private-address',
rid=rid, unit=unit)
if _laddr:
cluster_hosts[addr]['backends'][_unit] = _laddr
ctxt = {
'units': cluster_hosts,
'frontends': cluster_hosts,
}
if config('haproxy-server-timeout'):
@ -455,12 +490,13 @@ class HAProxyContext(OSContextGenerator):
ctxt['haproxy_host'] = '0.0.0.0'
ctxt['stat_port'] = ':8888'
if len(cluster_hosts.keys()) > 1:
# Enable haproxy when we have enough peers.
log('Ensuring haproxy enabled in /etc/default/haproxy.')
with open('/etc/default/haproxy', 'w') as out:
out.write('ENABLED=1\n')
return ctxt
for frontend in cluster_hosts:
if len(cluster_hosts[frontend]['backends']) > 1:
# Enable haproxy when we have enough peers.
log('Ensuring haproxy enabled in /etc/default/haproxy.')
with open('/etc/default/haproxy', 'w') as out:
out.write('ENABLED=1\n')
return ctxt
log('HAProxy context is incomplete, this unit has no peers.')
return {}
@ -722,22 +758,22 @@ class NeutronContext(OSContextGenerator):
class OSConfigFlagContext(OSContextGenerator):
"""
Responsible for adding user-defined config-flags in charm config to a
template context.
"""
Responsible for adding user-defined config-flags in charm config to a
template context.
NOTE: the value of config-flags may be a comma-separated list of
key=value pairs and some Openstack config files support
comma-separated lists as values.
"""
NOTE: the value of config-flags may be a comma-separated list of
key=value pairs and some Openstack config files support
comma-separated lists as values.
"""
def __call__(self):
config_flags = config('config-flags')
if not config_flags:
return {}
def __call__(self):
config_flags = config('config-flags')
if not config_flags:
return {}
flags = config_flags_parser(config_flags)
return {'user_config_flags': flags}
flags = config_flags_parser(config_flags)
return {'user_config_flags': flags}
class SubordinateConfigContext(OSContextGenerator):

20
hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
View File

@ -34,17 +34,21 @@ listen stats {{ stat_port }}
stats uri /
stats auth admin:password
{% if units -%}
{% if frontends -%}
{% for service, ports in service_ports.iteritems() -%}
listen {{ service }}_ipv4 0.0.0.0:{{ ports[0] }}
balance roundrobin
{% for unit, address in units.iteritems() -%}
server {{ unit }} {{ address }}:{{ ports[1] }} check
frontend tcp-in_{{ service }}
bind *:{{ ports[0] }}
bind :::{{ ports[0] }}
{% for frontend in frontends -%}
acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}
{% endfor %}
listen {{ service }}_ipv6 :::{{ ports[0] }}
balance roundrobin
{% for unit, address in units.iteritems() -%}
{% for frontend in frontends -%}
backend {{ service }}_{{ frontend }}
balance leastconn
{% for unit, address in frontends[frontend]['backends'].iteritems() -%}
server {{ unit }} {{ address }}:{{ ports[1] }} check
{% endfor %}
{% endfor -%}
{% endfor -%}
{% endif -%}

2
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -78,6 +78,8 @@ SWIFT_CODENAMES = OrderedDict([
('1.12.0', 'icehouse'),
('1.11.0', 'icehouse'),
('2.0.0', 'juno'),
('2.1.0', 'juno'),
('2.2.0', 'juno'),
])
DEFAULT_LOOPBACK_SIZE = '5G'

10
tests/00-setup
View File

@ -4,8 +4,8 @@ set -ex
sudo add-apt-repository --yes ppa:juju/stable
sudo apt-get update --yes
sudo apt-get install --yes python-amulet
sudo apt-get install --yes python-swiftclient
sudo apt-get install --yes python-glanceclient
sudo apt-get install --yes python-keystoneclient
sudo apt-get install --yes python-novaclient
sudo apt-get install --yes python-amulet \
python-swiftclient \
python-glanceclient \
python-keystoneclient \
python-novaclient

6
tests/README
View File

@ -1,6 +1,12 @@
This directory provides Amulet tests that focus on verification of swift-storage
deployments.
In order to run tests, you'll need charm-tools installed (in addition to
juju, of course):
sudo add-apt-repository ppa:juju/stable
sudo apt-get update
sudo apt-get install charm-tools
If you use a web proxy server to access the web, you'll need to set the
AMULET_HTTP_PROXY environment variable to the http URL of the proxy server.

30
tests/basic_deployment.py
View File

@ -20,10 +20,10 @@ u = OpenStackAmuletUtils(ERROR)
class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
"""Amulet tests on a basic swift-storage deployment."""
def __init__(self, series, openstack=None, source=None):
def __init__(self, series, openstack=None, source=None, stable=False):
"""Deploy the entire test environment."""
super(SwiftStorageBasicDeployment, self).__init__(series, openstack,
source)
source, stable)
self._add_services()
self._add_relations()
self._configure_services()
@ -31,12 +31,15 @@ class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
self._initialize_tests()
def _add_services(self):
"""Add the service that we're testing, including the number of units,
where swift-storage is local, and the other charms are from
the charm store."""
this_service = ('swift-storage', 1)
other_services = [('mysql', 1),
('keystone', 1), ('glance', 1), ('swift-proxy', 1)]
"""Add services
Add the services that we're testing, where swift-storage is local,
and the rest of the service are from lp branches that are
compatible with the local charm (e.g. stable or next).
"""
this_service = {'name': 'swift-storage'}
other_services = [{'name': 'mysql'}, {'name': 'keystone'},
{'name': 'glance'}, {'name': 'swift-proxy'}]
super(SwiftStorageBasicDeployment, self)._add_services(this_service,
other_services)
@ -249,9 +252,14 @@ class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
message = u.relation_error('swift-proxy swift-storage', ret)
amulet.raise_status(amulet.FAIL, msg=message)
def test_restart_on_config_change(self):
def test_z_restart_on_config_change(self):
"""Verify that the specified services are restarted when the config
is changed."""
is changed.
Note(coreycb): The method name with the _z_ is a little odd
but it forces the test to run last. It just makes things
easier because restarting services requires re-authorization.
"""
# NOTE(coreycb): Skipping failing test on until resolved. This test
# fails because the config file's last mod time is
# slightly after the process' last mod time.
@ -282,6 +290,8 @@ class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
config = '/etc/swift/{}'.format(conf)
if not u.service_restarted(self.swift_storage_sentry, s, config,
pgrep_full=True, sleep_time=time):
self.d.configure('swift-storage',
{'object-server-threads-per-disk': '4'})
msg = "service {} didn't restart after config change".format(s)
amulet.raise_status(amulet.FAIL, msg=msg)
time = 0

25
tests/charmhelpers/contrib/amulet/deployment.py
View File

@ -25,25 +25,30 @@ class AmuletDeployment(object):
Add services to the deployment where this_service is the local charm
that we're testing and other_services are the other services that
are being used in the amulet tests.
are being used in the local amulet tests.
"""
name, units, location = range(3)
if this_service[name] != os.path.basename(os.getcwd()):
s = this_service[name]
if this_service['name'] != os.path.basename(os.getcwd()):
s = this_service['name']
msg = "The charm's root directory name needs to be {}".format(s)
amulet.raise_status(amulet.FAIL, msg=msg)
self.d.add(this_service[name], units=this_service[units])
if 'units' not in this_service:
this_service['units'] = 1
self.d.add(this_service['name'], units=this_service['units'])
for svc in other_services:
if len(svc) > 2:
branch_location = svc[location]
if 'location' in svc:
branch_location = svc['location']
elif self.series:
branch_location = 'cs:{}/{}'.format(self.series, svc[name]),
branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
else:
branch_location = None
self.d.add(svc[name], charm=branch_location, units=svc[units])
if 'units' not in svc:
svc['units'] = 1
self.d.add(svc['name'], charm=branch_location, units=svc['units'])
def _add_relations(self, relations):
"""Add all of the relations for the services."""

61
tests/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@ -1,6 +1,3 @@
from bzrlib.branch import Branch
import os
import re
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
@ -13,62 +10,62 @@ class OpenStackAmuletDeployment(AmuletDeployment):
that is specifically for use by OpenStack charms.
"""
def __init__(self, series=None, openstack=None, source=None):
def __init__(self, series=None, openstack=None, source=None, stable=True):
"""Initialize the deployment environment."""
super(OpenStackAmuletDeployment, self).__init__(series)
self.openstack = openstack
self.source = source
def _is_dev_branch(self):
"""Determine if branch being tested is a dev (i.e. next) branch."""
branch = Branch.open(os.getcwd())
parent = branch.get_parent()
pattern = re.compile("^.*/next/$")
if (pattern.match(parent)):
return True
else:
return False
self.stable = stable
# Note(coreycb): this needs to be changed when new next branches come
# out.
self.current_next = "trusty"
def _determine_branch_locations(self, other_services):
"""Determine the branch locations for the other services.
If the branch being tested is a dev branch, then determine the
development branch locations for the other services. Otherwise,
the default charm store branches will be used."""
name = 0
if self._is_dev_branch():
updated_services = []
Determine if the local branch being tested is derived from its
stable or next (dev) branch, and based on this, use the corresonding
stable or next branches for the other_services."""
base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
if self.stable:
for svc in other_services:
if svc[name] in ['mysql', 'mongodb', 'rabbitmq-server']:
location = 'lp:charms/{}'.format(svc[name])
temp = 'lp:charms/{}'
svc['location'] = temp.format(svc['name'])
else:
for svc in other_services:
if svc['name'] in base_charms:
temp = 'lp:charms/{}'
svc['location'] = temp.format(svc['name'])
else:
temp = 'lp:~openstack-charmers/charms/trusty/{}/next'
location = temp.format(svc[name])
updated_services.append(svc + (location,))
other_services = updated_services
temp = 'lp:~openstack-charmers/charms/{}/{}/next'
svc['location'] = temp.format(self.current_next,
svc['name'])
return other_services
def _add_services(self, this_service, other_services):
"""Add services to the deployment and set openstack-origin/source."""
name = 0
other_services = self._determine_branch_locations(other_services)
super(OpenStackAmuletDeployment, self)._add_services(this_service,
other_services)
services = other_services
services.append(this_service)
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph']
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
'ceph-osd', 'ceph-radosgw']
if self.openstack:
for svc in services:
if svc[name] not in use_source:
if svc['name'] not in use_source:
config = {'openstack-origin': self.openstack}
self.d.configure(svc[name], config)
self.d.configure(svc['name'], config)
if self.source:
for svc in services:
if svc[name] in use_source:
if svc['name'] in use_source:
config = {'source': self.source}
self.d.configure(svc[name], config)
self.d.configure(svc['name'], config)
def _configure_services(self, configs):
"""Configure all of the services."""