Fix up port handling, checkin Makefile+setup.cfg+.coveragerc, rename tests/ -> unit_tests/

charm-helpers.yaml

@@ -0,0 +1,10 @@
+branch: lp:charm-helpers
+destination: hooks/charmhelpers
+include:
+    - core
+    - contrib.openstack|inc=*
+    - contrib.storage
+    - contrib.hahelpers:
+        - apache
+        - ceph
+        - cluster

hooks/misc_utils.py

@@ -0,0 +1,84 @@
+from charmhelpers.contrib.storage.linux.utils import (
+    is_block_device,
+    zap_disk,
+)
+
+from charmhelpers.contrib.storage.linux.loopback import (
+    ensure_loopback_device,
+)
+
+from charmhelpers.contrib.storage.linux.lvm import (
+    deactivate_lvm_volume_group,
+    is_lvm_physical_volume,
+    remove_lvm_physical_volume,
+)
+
+from charmhelpers.core.host import (
+    mounts,
+    umount,
+)
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO,
+    ERROR,
+)
+
+DEFAULT_LOOPBACK_SIZE = '5G'
+
+
+def ensure_block_device(block_device):
+    '''
+    Confirm block_device, create as loopback if necessary.
+
+    :param block_device: str: Full path of block device to ensure.
+
+    :returns: str: Full path of ensured block device.
+    '''
+    _none = ['None', 'none', None]
+    if (block_device in _none):
+        log('prepare_storage(): Missing required input: '
+            'block_device=%s.' % block_device, level=ERROR)
+        raise
+
+    if block_device.startswith('/dev/'):
+        bdev = block_device
+    elif block_device.startswith('/'):
+        _bd = block_device.split('|')
+        if len(_bd) == 2:
+            bdev, size = _bd
+        else:
+            bdev = block_device
+            size = DEFAULT_LOOPBACK_SIZE
+        bdev = ensure_loopback_device(bdev, size)
+    else:
+        bdev = '/dev/%s' % block_device
+
+    if not is_block_device(bdev):
+        log('Failed to locate valid block device at %s' % bdev, level=ERROR)
+        raise
+
+    return bdev
+
+
+def clean_storage(block_device):
+    '''
+    Ensures a block device is clean.  That is:
+        - unmounted
+        - any lvm volume groups are deactivated
+        - any lvm physical device signatures removed
+        - partition table wiped
+
+    :param block_device: str: Full path to block device to clean.
+    '''
+    for mp, d in mounts():
+        if d == block_device:
+            log('clean_storage(): Found %s mounted @ %s, unmounting.' %
+                (d, mp), level=INFO)
+            umount(mp, persist=True)
+
+    if is_lvm_physical_volume(block_device):
+        deactivate_lvm_volume_group(block_device)
+        remove_lvm_physical_volume(block_device)
+    else:
+        zap_disk(block_device)

hooks/swift_storage_context.py

@@ -0,0 +1,72 @@
+import re
+
+from charmhelpers.core.hookenv import (
+    config,
+    log,
+    related_units,
+    relation_get,
+    relation_ids,
+)
+
+from charmhelpers.contrib.openstack.utils import (
+    get_host_ip,
+)
+
+from charmhelpers.contrib.openstack.context import (
+    OSContextGenerator,
+)
+
+
+class SwiftStorageContext(OSContextGenerator):
+    interfaces = ['swift-storage']
+
+    def __call__(self):
+        rids = relation_ids('swift-storage')
+        if not rids:
+            return {}
+
+        swift_hash = None
+        for rid in rids:
+            for unit in related_units(rid):
+                if not swift_hash:
+                    swift_hash = relation_get('swift_hash', rid=rid,
+                                              unit=unit)
+        if not swift_hash:
+            log('No swift_hash passed via swift-storage relation. '
+                'Peer not ready?')
+            return {}
+        return {'swift_hash': swift_hash}
+
+
+class RsyncContext(OSContextGenerator):
+    interfaces = []
+
+    def enable_rsyncd(self):
+        default = open('/etc/default/rsync').read()
+        _m = re.compile('^RSYNC_ENABLE=(.*)$', re.MULTILINE)
+        if not re.search(_m, default):
+            with open('/etc/default/rsync', 'a+') as out:
+                out.write('RSYNC_ENABLE=true\n')
+        else:
+            with open('/etc/default/rsync', 'w') as out:
+                out.write(_m.sub('RSYNC_ENABLE=true', default))
+
+    def __call__(self):
+        local_ip = get_host_ip()
+        self.enable_rsyncd()
+        return {
+            'local_ip': local_ip
+        }
+
+
+class SwiftStorageServerContext(OSContextGenerator):
+    interfaces = []
+
+    def __call__(self):
+        ctxt = {
+            'local_ip': get_host_ip(),
+            'account_server_port': config('account-server-port'),
+            'container_server_port': config('container-server-port'),
+            'object_server_port': config('object-server-port'),
+        }
+        return ctxt

hooks/swift_storage_relations.py

@@ -39,13 +39,11 @@ hooks = Hooks()
 CONFIGS = register_configs()
 
 
-@hooks.hook('install')
-@restart_on_change(RESTART_MAP)
+@hooks.hook()
 def install():
     configure_installation_source(config('openstack-origin'))
     apt_update()
     apt_install(PACKAGES, fatal=True)
-    CONFIGS.write('/etc/rsyncd.conf')
     setup_storage()
     ensure_swift_directories()
 

revision

@@ -1 +1 @@
-87
+88

templates/account-server.conf

@@ -0,0 +1,21 @@
+[DEFAULT]
+bind_ip =  {{ local_ip }}
+bind_port = {{ account_server_port }}
+workers = 2
+
+[pipeline:main]
+pipeline = recon account-server
+
+[filter:recon]
+use = egg:swift#recon
+recon_cache_path = /var/cache/swift
+
+[app:account-server]
+use = egg:swift#account
+
+[account-replicator]
+
+[account-auditor]
+
+[account-reaper]
+

templates/container-server.conf

@@ -0,0 +1,23 @@
+[DEFAULT]
+bind_ip =  {{ local_ip }}
+bind_port = {{ container_server_port }}
+workers = 2
+
+[pipeline:main]
+pipeline = recon container-server
+
+[filter:recon]
+use = egg:swift#recon
+recon_cache_path = /var/cache/swift
+
+[app:container-server]
+use = egg:swift#container
+
+[container-replicator]
+
+[container-updater]
+
+[container-auditor]
+
+[container-sync]
+

templates/object-server.conf

@@ -0,0 +1,23 @@
+[DEFAULT]
+bind_ip =  {{ local_ip }}
+bind_port = {{ object_server_port }}
+workers = 2
+
+[pipeline:main]
+pipeline = recon object-server
+
+[filter:recon]
+use = egg:swift#recon
+recon_cache_path = /var/cache/swift
+
+[app:object-server]
+use = egg:swift#object
+
+[object-replicator]
+
+[object-updater]
+
+[object-auditor]
+
+[object-sync]
+

templates/rsyncd.conf

@@ -0,0 +1,23 @@
+uid = swift
+gid = swift
+log file = /var/log/rsyncd.log
+pid file = /var/run/rsyncd.pid
+address = {{ local_ip }}
+
+[account]
+max connections = 2
+path = /srv/node/
+read only = false
+lock file = /var/lock/account.lock
+
+[container]
+max connections = 2
+path = /srv/node/
+read only = false
+lock file = /var/lock/container.lock
+
+[object]
+max connections = 2
+path = /srv/node/
+read only = false
+lock file = /var/lock/object.lock

templates/swift.conf

@@ -0,0 +1,5 @@
+{% if swift_hash %}
+[swift-hash]
+# random unique string that can never change (DO NOT LOSE)
+swift_hash_path_suffix = {{ swift_hash }}
+{% endif %}

unit_tests/test_swift_storage_relations.py

@@ -0,0 +1,102 @@
+from mock import patch, MagicMock
+
+from unit_tests.test_utils import CharmTestCase
+
+import hooks.swift_storage_utils as utils
+
+_reg = utils.register_configs
+utils.register_configs = MagicMock()
+
+import hooks.swift_storage_relations as relations
+
+utils.register_configs = _reg
+
+from hooks.swift_storage_utils import PACKAGES
+
+TO_PATCH = [
+    'CONFIGS',
+    # charmhelpers.core.hookenv
+    'Hooks',
+    'config',
+    'log',
+    'relation_set',
+    'relation_get',
+    # charmhelpers.core.host
+    'apt_update',
+    'apt_install',
+    # charmehelpers.contrib.openstack.utils
+    'configure_installation_source',
+    'openstack_upgrade_available',
+    # swift_storage_utils
+    'determine_block_devices',
+    'do_openstack_upgrade',
+    'ensure_swift_directories',
+    'fetch_swift_rings',
+    'save_script_rc',
+    'setup_storage',
+    'register_configs',
+]
+
+
+class SwiftStorageRelationsTests(CharmTestCase):
+    def setUp(self):
+        super(SwiftStorageRelationsTests, self).setUp(relations,
+                                                      TO_PATCH)
+        self.config.side_effect = self.test_config.get
+        self.relation_get.side_effect = self.test_relation.get
+
+    def test_install_hook(self):
+        self.test_config.set('openstack-origin', 'cloud:precise-havana')
+        relations.install()
+        self.configure_installation_source.assert_called_with(
+            'cloud:precise-havana',
+        )
+        self.apt_update.assert_called()
+        self.apt_install.assert_called_with(PACKAGES, fatal=True)
+
+        self.setup_storage.assert_called()
+
+    def test_config_changed_no_upgrade_available(self):
+        self.openstack_upgrade_available.return_value = False
+        relations.config_changed()
+        self.assertFalse(self.do_openstack_upgrade.called)
+        self.assertTrue(self.CONFIGS.write_all.called)
+
+    def test_config_changed_upgrade_available(self):
+        self.openstack_upgrade_available.return_value = True
+        relations.config_changed()
+        self.assertTrue(self.do_openstack_upgrade.called)
+        self.assertTrue(self.CONFIGS.write_all.called)
+
+    def test_storage_joined_single_device(self):
+        self.determine_block_devices.return_value = ['/dev/vdb']
+        relations.swift_storage_relation_joined()
+        self.relation_set.assert_called_with(
+            device='vdb', object_port=6000, account_port=6002,
+            zone=1, container_port=6001
+        )
+
+    def test_storage_joined_multi_device(self):
+        self.determine_block_devices.return_value = ['/dev/vdb', '/dev/vdc',
+                                                     '/dev/vdd']
+        relations.swift_storage_relation_joined()
+        self.relation_set.assert_called_with(
+            device='vdb:vdc:vdd', object_port=6000, account_port=6002,
+            zone=1, container_port=6001
+        )
+
+    @patch('sys.exit')
+    def test_storage_changed_missing_relation_data(self, exit):
+        relations.swift_storage_relation_changed()
+        exit.assert_called_with(0)
+
+    def test_storage_changed_with_relation_data(self):
+        self.test_relation.set({
+            'swift_hash': 'foo_hash',
+            'rings_url': 'http://swift-proxy.com/rings/',
+        })
+        relations.swift_storage_relation_changed()
+        self.CONFIGS.write.assert_called_with('/etc/swift/swift.conf')
+        self.fetch_swift_rings.assert_called_with(
+            'http://swift-proxy.com/rings/'
+        )

unit_tests/test_swift_storage_utils.py

@@ -0,0 +1,187 @@
+from mock import call, patch, MagicMock
+from contextlib import contextmanager
+from unit_tests.test_utils import CharmTestCase
+
+
+import hooks.swift_storage_utils as swift_utils
+
+
+TO_PATCH = [
+    'log',
+    'config',
+    'mkdir',
+    'mount',
+    'check_call',
+    'call',
+    'ensure_block_device',
+    'clean_storage',
+    'is_block_device',
+    'get_os_codename_package',
+    'get_host_ip',
+    '_save_script_rc',
+]
+
+
+PROC_PARTITIONS = """
+major minor  #blocks  name
+
+   8        0  732574584 sda
+   8        1     102400 sda1
+   8        2  307097600 sda2
+   8        3          1 sda3
+   8        5  146483200 sda5
+   8        6    4881408 sda6
+   8        7  274004992 sda7
+   8       16  175825944 sdb
+   9        0  732574584 vda
+  10        0  732574584 vdb
+  10        0  732574584 vdb1
+ 104        0 1003393784 cciss/c0d0
+ 105        0 1003393784 cciss/c1d0
+ 105        1   86123689 cciss/c1d0p1
+ 252        0   20971520 dm-0
+ 252        1   15728640 dm-1
+"""
+
+SCRIPT_RC_ENV = {
+    'OPENSTACK_PORT_ACCOUNT': 6002,
+    'OPENSTACK_PORT_CONTAINER': 6001,
+    'OPENSTACK_PORT_OBJECT': 6000,
+    'OPENSTACK_SWIFT_SERVICE_ACCOUNT': 'account-server',
+    'OPENSTACK_SWIFT_SERVICE_CONTAINER': 'container-server',
+    'OPENSTACK_SWIFT_SERVICE_OBJECT': 'object-server',
+    'OPENSTACK_URL_ACCOUNT':
+    'http://10.0.0.1:6002/recon/diskusage|"mounted":true',
+    'OPENSTACK_URL_CONTAINER':
+    'http://10.0.0.1:6001/recon/diskusage|"mounted":true',
+    'OPENSTACK_URL_OBJECT':
+    'http://10.0.0.1:6000/recon/diskusage|"mounted":true'
+}
+
+
+@contextmanager
+def patch_open():
+    '''Patch open() to allow mocking both open() itself and the file that is
+    yielded.
+
+    Yields the mock for "open" and "file", respectively.'''
+    mock_open = MagicMock(spec=open)
+    mock_file = MagicMock(spec=file)
+
+    @contextmanager
+    def stub_open(*args, **kwargs):
+        mock_open(*args, **kwargs)
+        yield mock_file
+
+    with patch('__builtin__.open', stub_open):
+        yield mock_open, mock_file
+
+
+class SwiftStorageUtilsTests(CharmTestCase):
+    def setUp(self):
+        super(SwiftStorageUtilsTests, self).setUp(swift_utils, TO_PATCH)
+        self.config.side_effect = self.test_config.get
+
+    def test_ensure_swift_directories(self):
+        with patch('os.path.isdir') as isdir:
+            isdir.return_value = False
+            swift_utils.ensure_swift_directories()
+            ex_dirs = [
+                call('/etc/swift', owner='swift', group='swift'),
+                call('/var/cache/swift', owner='swift', group='swift'),
+                call('/srv/node', owner='swift', group='swift')
+            ]
+        self.assertEquals(ex_dirs, self.mkdir.call_args_list)
+
+    def test_swift_init_nonfatal(self):
+        swift_utils.swift_init('all', 'start')
+        self.call.assert_called_with(['swift-init', 'all', 'start'])
+
+    def test_swift_init_fatal(self):
+        swift_utils.swift_init('all', 'start', fatal=True)
+        self.check_call.assert_called_with(['swift-init', 'all', 'start'])
+
+    def test_fetch_swift_rings(self):
+        url = 'http://someproxynode/rings'
+        swift_utils.fetch_swift_rings(url)
+        wgets = []
+        for s in ['account', 'object', 'container']:
+            _c = call(['wget', '%s/%s.ring.gz' % (url, s),
+                      '-O', '/etc/swift/%s.ring.gz' % s])
+            wgets.append(_c)
+        self.assertEquals(wgets, self.check_call.call_args_list)
+
+    def test_determine_block_device_no_config(self):
+        self.test_config.set('block-device', None)
+        self.assertEquals(swift_utils.determine_block_devices(), None)
+
+    def _fake_ensure(self, bdev):
+        return bdev.split('|').pop(0)
+
+    @patch.object(swift_utils, 'ensure_block_device')
+    def test_determine_block_device_single_dev(self, _ensure):
+        _ensure.side_effect = self._fake_ensure
+        self.test_config.set('block-device', '/dev/vdb')
+        result = swift_utils.determine_block_devices()
+        self.assertEquals(['/dev/vdb'], result)
+
+    @patch.object(swift_utils, 'ensure_block_device')
+    def test_determine_block_device_multi_dev(self, _ensure):
+        _ensure.side_effect = self._fake_ensure
+        bdevs = '/dev/vdb /dev/vdc /tmp/swift.img|1G'
+        self.test_config.set('block-device', bdevs)
+        result = swift_utils.determine_block_devices()
+        ex = ['/dev/vdb', '/dev/vdc', '/tmp/swift.img']
+        self.assertEquals(ex, result)
+
+    @patch.object(swift_utils, 'find_block_devices')
+    @patch.object(swift_utils, 'ensure_block_device')
+    def test_determine_block_device_guess_dev(self, _ensure, _find):
+        _ensure.side_effect = self._fake_ensure
+        self.test_config.set('block-device', 'guess')
+        _find.return_value = ['/dev/vdb', '/dev/sdb']
+        result = swift_utils.determine_block_devices()
+        self.assertTrue(_find.called)
+        self.assertEquals(result, ['/dev/vdb', '/dev/sdb'])
+
+    def test_mkfs_xfs(self):
+        swift_utils.mkfs_xfs('/dev/sdb')
+        self.check_call.assert_called_with(
+            ['mkfs.xfs', '-f', '-i', 'size=1024', '/dev/sdb']
+        )
+
+    @patch.object(swift_utils, 'clean_storage')
+    @patch.object(swift_utils, 'mkfs_xfs')
+    @patch.object(swift_utils, 'determine_block_devices')
+    def test_setup_storage_no_overwrite(self, determine, mkfs, clean):
+        determine.return_value = ['/dev/vdb']
+        self.test_config.set('overwrite', 'false')
+        swift_utils.setup_storage()
+        self.assertFalse(clean.called)
+
+    @patch.object(swift_utils, 'clean_storage')
+    @patch.object(swift_utils, 'mkfs_xfs')
+    @patch.object(swift_utils, 'determine_block_devices')
+    def test_setup_storage_overwrite(self, determine, mkfs, clean):
+        determine.return_value = ['/dev/vdb']
+        self.test_config.set('overwrite', 'True')
+        swift_utils.setup_storage()
+        clean.assert_called_with('/dev/vdb')
+        self.mkdir.assert_called_with('/srv/node/vdb', owner='swift',
+                                      group='swift')
+        self.mount.assert_called('/dev/vdb', '/srv/node/vdb', persist=True)
+
+    def test_find_block_devices(self):
+        self.is_block_device.return_value = True
+        with patch_open() as (_open, _file):
+            _file.read.return_value = PROC_PARTITIONS
+            _file.readlines = MagicMock()
+            _file.readlines.return_value = PROC_PARTITIONS.split('\n')
+            result = swift_utils.find_block_devices()
+        ex = ['/dev/sdb', '/dev/vdb', '/dev/cciss/c1d0']
+        self.assertEquals(ex, result)
+
+    def test_save_script_rc(self):
+        self.get_host_ip.return_value = '10.0.0.1'
+        swift_utils.save_script_rc()
+        self._save_script_rc.assert_called_with(**SCRIPT_RC_ENV)

unit_tests/test_utils.py

@@ -0,0 +1,99 @@
+import logging
+import unittest
+import os
+import yaml
+
+from mock import patch
+
+
+def load_config():
+    '''
+    Walk backwords from __file__ looking for config.yaml, load and return the
+    'options' section'
+    '''
+    config = None
+    f = __file__
+    while config is None:
+        d = os.path.dirname(f)
+        if os.path.isfile(os.path.join(d, 'config.yaml')):
+            config = os.path.join(d, 'config.yaml')
+            break
+        f = d
+
+    if not config:
+        logging.error('Could not find config.yaml in any parent directory '
+                      'of %s. ' % file)
+        raise Exception
+
+    return yaml.safe_load(open(config).read())['options']
+
+
+def get_default_config():
+    '''
+    Load default charm config from config.yaml return as a dict.
+    If no default is set in config.yaml, its value is None.
+    '''
+    default_config = {}
+    config = load_config()
+    for k, v in config.iteritems():
+        if 'default' in v:
+            default_config[k] = v['default']
+        else:
+            default_config[k] = None
+    return default_config
+
+
+class CharmTestCase(unittest.TestCase):
+    def setUp(self, obj, patches):
+        super(CharmTestCase, self).setUp()
+        self.patches = patches
+        self.obj = obj
+        self.test_config = TestConfig()
+        self.test_relation = TestRelation()
+        self.patch_all()
+
+    def patch(self, method):
+        _m = patch.object(self.obj, method)
+        mock = _m.start()
+        self.addCleanup(_m.stop)
+        return mock
+
+    def patch_all(self):
+        for method in self.patches:
+            setattr(self, method, self.patch(method))
+
+
+class TestConfig(object):
+    def __init__(self):
+        self.config = get_default_config()
+
+    def get(self, attr=None):
+        if not attr:
+            return self.get_all()
+        try:
+            return self.config[attr]
+        except KeyError:
+            return None
+
+    def get_all(self):
+        return self.config
+
+    def set(self, attr, value):
+            if attr not in self.config:
+                raise KeyError
+            self.config[attr] = value
+
+
+class TestRelation(object):
+    def __init__(self, relation_data={}):
+        self.relation_data = relation_data
+
+    def set(self, relation_data):
+        self.relation_data = relation_data
+
+    def get(self, attr=None, unit=None, rid=None):
+        if attr is None:
+            return self.relation_data
+        elif attr in self.relation_data:
+            return self.relation_data[attr]
+        return None