Fix up port handling, checkin Makefile+setup.cfg+.coveragerc, rename tests/ -> unit_tests/

This commit is contained in:
Adam Gandelman 2013-07-19 12:52:45 -07:00
parent 85501cbb64
commit b99f2cf010
14 changed files with 651 additions and 4 deletions

10
charm-helpers.yaml Normal file
View File

@ -0,0 +1,10 @@
branch: lp:charm-helpers
destination: hooks/charmhelpers
include:
- core
- contrib.openstack|inc=*
- contrib.storage
- contrib.hahelpers:
- apache
- ceph
- cluster

84
hooks/misc_utils.py Normal file
View File

@ -0,0 +1,84 @@
from charmhelpers.contrib.storage.linux.utils import (
is_block_device,
zap_disk,
)
from charmhelpers.contrib.storage.linux.loopback import (
ensure_loopback_device,
)
from charmhelpers.contrib.storage.linux.lvm import (
deactivate_lvm_volume_group,
is_lvm_physical_volume,
remove_lvm_physical_volume,
)
from charmhelpers.core.host import (
mounts,
umount,
)
from charmhelpers.core.hookenv import (
log,
INFO,
ERROR,
)
DEFAULT_LOOPBACK_SIZE = '5G'
def ensure_block_device(block_device):
'''
Confirm block_device, create as loopback if necessary.
:param block_device: str: Full path of block device to ensure.
:returns: str: Full path of ensured block device.
'''
_none = ['None', 'none', None]
if (block_device in _none):
log('prepare_storage(): Missing required input: '
'block_device=%s.' % block_device, level=ERROR)
raise
if block_device.startswith('/dev/'):
bdev = block_device
elif block_device.startswith('/'):
_bd = block_device.split('|')
if len(_bd) == 2:
bdev, size = _bd
else:
bdev = block_device
size = DEFAULT_LOOPBACK_SIZE
bdev = ensure_loopback_device(bdev, size)
else:
bdev = '/dev/%s' % block_device
if not is_block_device(bdev):
log('Failed to locate valid block device at %s' % bdev, level=ERROR)
raise
return bdev
def clean_storage(block_device):
'''
Ensures a block device is clean. That is:
- unmounted
- any lvm volume groups are deactivated
- any lvm physical device signatures removed
- partition table wiped
:param block_device: str: Full path to block device to clean.
'''
for mp, d in mounts():
if d == block_device:
log('clean_storage(): Found %s mounted @ %s, unmounting.' %
(d, mp), level=INFO)
umount(mp, persist=True)
if is_lvm_physical_volume(block_device):
deactivate_lvm_volume_group(block_device)
remove_lvm_physical_volume(block_device)
else:
zap_disk(block_device)

View File

@ -0,0 +1,72 @@
import re
from charmhelpers.core.hookenv import (
config,
log,
related_units,
relation_get,
relation_ids,
)
from charmhelpers.contrib.openstack.utils import (
get_host_ip,
)
from charmhelpers.contrib.openstack.context import (
OSContextGenerator,
)
class SwiftStorageContext(OSContextGenerator):
interfaces = ['swift-storage']
def __call__(self):
rids = relation_ids('swift-storage')
if not rids:
return {}
swift_hash = None
for rid in rids:
for unit in related_units(rid):
if not swift_hash:
swift_hash = relation_get('swift_hash', rid=rid,
unit=unit)
if not swift_hash:
log('No swift_hash passed via swift-storage relation. '
'Peer not ready?')
return {}
return {'swift_hash': swift_hash}
class RsyncContext(OSContextGenerator):
interfaces = []
def enable_rsyncd(self):
default = open('/etc/default/rsync').read()
_m = re.compile('^RSYNC_ENABLE=(.*)$', re.MULTILINE)
if not re.search(_m, default):
with open('/etc/default/rsync', 'a+') as out:
out.write('RSYNC_ENABLE=true\n')
else:
with open('/etc/default/rsync', 'w') as out:
out.write(_m.sub('RSYNC_ENABLE=true', default))
def __call__(self):
local_ip = get_host_ip()
self.enable_rsyncd()
return {
'local_ip': local_ip
}
class SwiftStorageServerContext(OSContextGenerator):
interfaces = []
def __call__(self):
ctxt = {
'local_ip': get_host_ip(),
'account_server_port': config('account-server-port'),
'container_server_port': config('container-server-port'),
'object_server_port': config('object-server-port'),
}
return ctxt

View File

@ -39,13 +39,11 @@ hooks = Hooks()
CONFIGS = register_configs()
@hooks.hook('install')
@restart_on_change(RESTART_MAP)
@hooks.hook()
def install():
configure_installation_source(config('openstack-origin'))
apt_update()
apt_install(PACKAGES, fatal=True)
CONFIGS.write('/etc/rsyncd.conf')
setup_storage()
ensure_swift_directories()

View File

@ -1 +1 @@
87
88

View File

@ -0,0 +1,21 @@
[DEFAULT]
bind_ip = {{ local_ip }}
bind_port = {{ account_server_port }}
workers = 2
[pipeline:main]
pipeline = recon account-server
[filter:recon]
use = egg:swift#recon
recon_cache_path = /var/cache/swift
[app:account-server]
use = egg:swift#account
[account-replicator]
[account-auditor]
[account-reaper]

View File

@ -0,0 +1,23 @@
[DEFAULT]
bind_ip = {{ local_ip }}
bind_port = {{ container_server_port }}
workers = 2
[pipeline:main]
pipeline = recon container-server
[filter:recon]
use = egg:swift#recon
recon_cache_path = /var/cache/swift
[app:container-server]
use = egg:swift#container
[container-replicator]
[container-updater]
[container-auditor]
[container-sync]

View File

@ -0,0 +1,23 @@
[DEFAULT]
bind_ip = {{ local_ip }}
bind_port = {{ object_server_port }}
workers = 2
[pipeline:main]
pipeline = recon object-server
[filter:recon]
use = egg:swift#recon
recon_cache_path = /var/cache/swift
[app:object-server]
use = egg:swift#object
[object-replicator]
[object-updater]
[object-auditor]
[object-sync]

23
templates/rsyncd.conf Normal file
View File

@ -0,0 +1,23 @@
uid = swift
gid = swift
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid
address = {{ local_ip }}
[account]
max connections = 2
path = /srv/node/
read only = false
lock file = /var/lock/account.lock
[container]
max connections = 2
path = /srv/node/
read only = false
lock file = /var/lock/container.lock
[object]
max connections = 2
path = /srv/node/
read only = false
lock file = /var/lock/object.lock

5
templates/swift.conf Normal file
View File

@ -0,0 +1,5 @@
{% if swift_hash %}
[swift-hash]
# random unique string that can never change (DO NOT LOSE)
swift_hash_path_suffix = {{ swift_hash }}
{% endif %}

0
unit_tests/__init__.py Normal file
View File

View File

@ -0,0 +1,102 @@
from mock import patch, MagicMock
from unit_tests.test_utils import CharmTestCase
import hooks.swift_storage_utils as utils
_reg = utils.register_configs
utils.register_configs = MagicMock()
import hooks.swift_storage_relations as relations
utils.register_configs = _reg
from hooks.swift_storage_utils import PACKAGES
TO_PATCH = [
'CONFIGS',
# charmhelpers.core.hookenv
'Hooks',
'config',
'log',
'relation_set',
'relation_get',
# charmhelpers.core.host
'apt_update',
'apt_install',
# charmehelpers.contrib.openstack.utils
'configure_installation_source',
'openstack_upgrade_available',
# swift_storage_utils
'determine_block_devices',
'do_openstack_upgrade',
'ensure_swift_directories',
'fetch_swift_rings',
'save_script_rc',
'setup_storage',
'register_configs',
]
class SwiftStorageRelationsTests(CharmTestCase):
def setUp(self):
super(SwiftStorageRelationsTests, self).setUp(relations,
TO_PATCH)
self.config.side_effect = self.test_config.get
self.relation_get.side_effect = self.test_relation.get
def test_install_hook(self):
self.test_config.set('openstack-origin', 'cloud:precise-havana')
relations.install()
self.configure_installation_source.assert_called_with(
'cloud:precise-havana',
)
self.apt_update.assert_called()
self.apt_install.assert_called_with(PACKAGES, fatal=True)
self.setup_storage.assert_called()
def test_config_changed_no_upgrade_available(self):
self.openstack_upgrade_available.return_value = False
relations.config_changed()
self.assertFalse(self.do_openstack_upgrade.called)
self.assertTrue(self.CONFIGS.write_all.called)
def test_config_changed_upgrade_available(self):
self.openstack_upgrade_available.return_value = True
relations.config_changed()
self.assertTrue(self.do_openstack_upgrade.called)
self.assertTrue(self.CONFIGS.write_all.called)
def test_storage_joined_single_device(self):
self.determine_block_devices.return_value = ['/dev/vdb']
relations.swift_storage_relation_joined()
self.relation_set.assert_called_with(
device='vdb', object_port=6000, account_port=6002,
zone=1, container_port=6001
)
def test_storage_joined_multi_device(self):
self.determine_block_devices.return_value = ['/dev/vdb', '/dev/vdc',
'/dev/vdd']
relations.swift_storage_relation_joined()
self.relation_set.assert_called_with(
device='vdb:vdc:vdd', object_port=6000, account_port=6002,
zone=1, container_port=6001
)
@patch('sys.exit')
def test_storage_changed_missing_relation_data(self, exit):
relations.swift_storage_relation_changed()
exit.assert_called_with(0)
def test_storage_changed_with_relation_data(self):
self.test_relation.set({
'swift_hash': 'foo_hash',
'rings_url': 'http://swift-proxy.com/rings/',
})
relations.swift_storage_relation_changed()
self.CONFIGS.write.assert_called_with('/etc/swift/swift.conf')
self.fetch_swift_rings.assert_called_with(
'http://swift-proxy.com/rings/'
)

View File

@ -0,0 +1,187 @@
from mock import call, patch, MagicMock
from contextlib import contextmanager
from unit_tests.test_utils import CharmTestCase
import hooks.swift_storage_utils as swift_utils
TO_PATCH = [
'log',
'config',
'mkdir',
'mount',
'check_call',
'call',
'ensure_block_device',
'clean_storage',
'is_block_device',
'get_os_codename_package',
'get_host_ip',
'_save_script_rc',
]
PROC_PARTITIONS = """
major minor #blocks name
8 0 732574584 sda
8 1 102400 sda1
8 2 307097600 sda2
8 3 1 sda3
8 5 146483200 sda5
8 6 4881408 sda6
8 7 274004992 sda7
8 16 175825944 sdb
9 0 732574584 vda
10 0 732574584 vdb
10 0 732574584 vdb1
104 0 1003393784 cciss/c0d0
105 0 1003393784 cciss/c1d0
105 1 86123689 cciss/c1d0p1
252 0 20971520 dm-0
252 1 15728640 dm-1
"""
SCRIPT_RC_ENV = {
'OPENSTACK_PORT_ACCOUNT': 6002,
'OPENSTACK_PORT_CONTAINER': 6001,
'OPENSTACK_PORT_OBJECT': 6000,
'OPENSTACK_SWIFT_SERVICE_ACCOUNT': 'account-server',
'OPENSTACK_SWIFT_SERVICE_CONTAINER': 'container-server',
'OPENSTACK_SWIFT_SERVICE_OBJECT': 'object-server',
'OPENSTACK_URL_ACCOUNT':
'http://10.0.0.1:6002/recon/diskusage|"mounted":true',
'OPENSTACK_URL_CONTAINER':
'http://10.0.0.1:6001/recon/diskusage|"mounted":true',
'OPENSTACK_URL_OBJECT':
'http://10.0.0.1:6000/recon/diskusage|"mounted":true'
}
@contextmanager
def patch_open():
'''Patch open() to allow mocking both open() itself and the file that is
yielded.
Yields the mock for "open" and "file", respectively.'''
mock_open = MagicMock(spec=open)
mock_file = MagicMock(spec=file)
@contextmanager
def stub_open(*args, **kwargs):
mock_open(*args, **kwargs)
yield mock_file
with patch('__builtin__.open', stub_open):
yield mock_open, mock_file
class SwiftStorageUtilsTests(CharmTestCase):
def setUp(self):
super(SwiftStorageUtilsTests, self).setUp(swift_utils, TO_PATCH)
self.config.side_effect = self.test_config.get
def test_ensure_swift_directories(self):
with patch('os.path.isdir') as isdir:
isdir.return_value = False
swift_utils.ensure_swift_directories()
ex_dirs = [
call('/etc/swift', owner='swift', group='swift'),
call('/var/cache/swift', owner='swift', group='swift'),
call('/srv/node', owner='swift', group='swift')
]
self.assertEquals(ex_dirs, self.mkdir.call_args_list)
def test_swift_init_nonfatal(self):
swift_utils.swift_init('all', 'start')
self.call.assert_called_with(['swift-init', 'all', 'start'])
def test_swift_init_fatal(self):
swift_utils.swift_init('all', 'start', fatal=True)
self.check_call.assert_called_with(['swift-init', 'all', 'start'])
def test_fetch_swift_rings(self):
url = 'http://someproxynode/rings'
swift_utils.fetch_swift_rings(url)
wgets = []
for s in ['account', 'object', 'container']:
_c = call(['wget', '%s/%s.ring.gz' % (url, s),
'-O', '/etc/swift/%s.ring.gz' % s])
wgets.append(_c)
self.assertEquals(wgets, self.check_call.call_args_list)
def test_determine_block_device_no_config(self):
self.test_config.set('block-device', None)
self.assertEquals(swift_utils.determine_block_devices(), None)
def _fake_ensure(self, bdev):
return bdev.split('|').pop(0)
@patch.object(swift_utils, 'ensure_block_device')
def test_determine_block_device_single_dev(self, _ensure):
_ensure.side_effect = self._fake_ensure
self.test_config.set('block-device', '/dev/vdb')
result = swift_utils.determine_block_devices()
self.assertEquals(['/dev/vdb'], result)
@patch.object(swift_utils, 'ensure_block_device')
def test_determine_block_device_multi_dev(self, _ensure):
_ensure.side_effect = self._fake_ensure
bdevs = '/dev/vdb /dev/vdc /tmp/swift.img|1G'
self.test_config.set('block-device', bdevs)
result = swift_utils.determine_block_devices()
ex = ['/dev/vdb', '/dev/vdc', '/tmp/swift.img']
self.assertEquals(ex, result)
@patch.object(swift_utils, 'find_block_devices')
@patch.object(swift_utils, 'ensure_block_device')
def test_determine_block_device_guess_dev(self, _ensure, _find):
_ensure.side_effect = self._fake_ensure
self.test_config.set('block-device', 'guess')
_find.return_value = ['/dev/vdb', '/dev/sdb']
result = swift_utils.determine_block_devices()
self.assertTrue(_find.called)
self.assertEquals(result, ['/dev/vdb', '/dev/sdb'])
def test_mkfs_xfs(self):
swift_utils.mkfs_xfs('/dev/sdb')
self.check_call.assert_called_with(
['mkfs.xfs', '-f', '-i', 'size=1024', '/dev/sdb']
)
@patch.object(swift_utils, 'clean_storage')
@patch.object(swift_utils, 'mkfs_xfs')
@patch.object(swift_utils, 'determine_block_devices')
def test_setup_storage_no_overwrite(self, determine, mkfs, clean):
determine.return_value = ['/dev/vdb']
self.test_config.set('overwrite', 'false')
swift_utils.setup_storage()
self.assertFalse(clean.called)
@patch.object(swift_utils, 'clean_storage')
@patch.object(swift_utils, 'mkfs_xfs')
@patch.object(swift_utils, 'determine_block_devices')
def test_setup_storage_overwrite(self, determine, mkfs, clean):
determine.return_value = ['/dev/vdb']
self.test_config.set('overwrite', 'True')
swift_utils.setup_storage()
clean.assert_called_with('/dev/vdb')
self.mkdir.assert_called_with('/srv/node/vdb', owner='swift',
group='swift')
self.mount.assert_called('/dev/vdb', '/srv/node/vdb', persist=True)
def test_find_block_devices(self):
self.is_block_device.return_value = True
with patch_open() as (_open, _file):
_file.read.return_value = PROC_PARTITIONS
_file.readlines = MagicMock()
_file.readlines.return_value = PROC_PARTITIONS.split('\n')
result = swift_utils.find_block_devices()
ex = ['/dev/sdb', '/dev/vdb', '/dev/cciss/c1d0']
self.assertEquals(ex, result)
def test_save_script_rc(self):
self.get_host_ip.return_value = '10.0.0.1'
swift_utils.save_script_rc()
self._save_script_rc.assert_called_with(**SCRIPT_RC_ENV)

99
unit_tests/test_utils.py Normal file
View File

@ -0,0 +1,99 @@
import logging
import unittest
import os
import yaml
from mock import patch
def load_config():
'''
Walk backwords from __file__ looking for config.yaml, load and return the
'options' section'
'''
config = None
f = __file__
while config is None:
d = os.path.dirname(f)
if os.path.isfile(os.path.join(d, 'config.yaml')):
config = os.path.join(d, 'config.yaml')
break
f = d
if not config:
logging.error('Could not find config.yaml in any parent directory '
'of %s. ' % file)
raise Exception
return yaml.safe_load(open(config).read())['options']
def get_default_config():
'''
Load default charm config from config.yaml return as a dict.
If no default is set in config.yaml, its value is None.
'''
default_config = {}
config = load_config()
for k, v in config.iteritems():
if 'default' in v:
default_config[k] = v['default']
else:
default_config[k] = None
return default_config
class CharmTestCase(unittest.TestCase):
def setUp(self, obj, patches):
super(CharmTestCase, self).setUp()
self.patches = patches
self.obj = obj
self.test_config = TestConfig()
self.test_relation = TestRelation()
self.patch_all()
def patch(self, method):
_m = patch.object(self.obj, method)
mock = _m.start()
self.addCleanup(_m.stop)
return mock
def patch_all(self):
for method in self.patches:
setattr(self, method, self.patch(method))
class TestConfig(object):
def __init__(self):
self.config = get_default_config()
def get(self, attr=None):
if not attr:
return self.get_all()
try:
return self.config[attr]
except KeyError:
return None
def get_all(self):
return self.config
def set(self, attr, value):
if attr not in self.config:
raise KeyError
self.config[attr] = value
class TestRelation(object):
def __init__(self, relation_data={}):
self.relation_data = relation_data
def set(self, relation_data):
self.relation_data = relation_data
def get(self, attr=None, unit=None, rid=None):
if attr is None:
return self.relation_data
elif attr in self.relation_data:
return self.relation_data[attr]
return None