Add execd preinstall, tidy unit testing

charm-helpers.yaml

@@ -7,5 +7,5 @@ include:
     - fetch
     - contrib.hahelpers:
         - apache
-        - ceph
         - cluster
+    - payload.execd

hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf

@@ -0,0 +1,23 @@
+{% if endpoints -%}
+{% for ext, int in endpoints -%}
+Listen {{ ext }}
+NameVirtualHost *:{{ ext }}
+<VirtualHost *:{{ ext }}>
+    ServerName {{ private_address }}
+    SSLEngine on
+    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
+    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
+    ProxyPass / http://localhost:{{ int }}/
+    ProxyPassReverse / http://localhost:{{ int }}/
+    ProxyPreserveHost on
+</VirtualHost>
+<Proxy *>
+    Order deny,allow
+    Allow from all
+</Proxy>
+<Location />
+    Order allow,deny
+    Allow from all
+</Location>
+{% endfor -%}
+{% endif -%}

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -97,12 +97,13 @@ def pool_exists(service, name):
         return name in out
 
 
-def get_osds():
+def get_osds(service):
     '''
     Return a list of all Ceph Object Storage Daemons
     currently in the cluster
     '''
-    return json.loads(check_output(['ceph', 'osd', 'ls', '--format=json']))
+    return json.loads(check_output(['ceph', '--id', service,
+                                    'osd', 'ls', '--format=json']))
 
 
 def create_pool(service, name, replicas=2):
@@ -113,17 +114,17 @@ def create_pool(service, name, replicas=2):
         return
     # Calculate the number of placement groups based
     # on upstream recommended best practices.
-    pgnum = (len(get_osds()) * 100 / replicas)
+    pgnum = (len(get_osds(service)) * 100 / replicas)
     cmd = [
         'ceph', '--id', service,
         'osd', 'pool', 'create',
-        name, pgnum
+        name, str(pgnum)
     ]
     check_call(cmd)
     cmd = [
         'ceph', '--id', service,
-        'osd', 'set', name,
-        'size', replicas
+        'osd', 'pool', 'set', name,
+        'size', str(replicas)
     ]
     check_call(cmd)
 

hooks/charmhelpers/payload/__init__.py

@@ -0,0 +1 @@
+"Tools for working with files injected into a charm just before deployment."

hooks/charmhelpers/payload/execd.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import subprocess
+from charmhelpers.core import hookenv
+
+
+def default_execd_dir():
+    return os.path.join(os.environ['CHARM_DIR'], 'exec.d')
+
+
+def execd_module_paths(execd_dir=None):
+    """Generate a list of full paths to modules within execd_dir."""
+    if not execd_dir:
+        execd_dir = default_execd_dir()
+
+    if not os.path.exists(execd_dir):
+        return
+
+    for subpath in os.listdir(execd_dir):
+        module = os.path.join(execd_dir, subpath)
+        if os.path.isdir(module):
+            yield module
+
+
+def execd_submodule_paths(command, execd_dir=None):
+    """Generate a list of full paths to the specified command within exec_dir.
+    """
+    for module_path in execd_module_paths(execd_dir):
+        path = os.path.join(module_path, command)
+        if os.access(path, os.X_OK) and os.path.isfile(path):
+            yield path
+
+
+def execd_run(command, execd_dir=None, die_on_error=False, stderr=None):
+    """Run command for each module within execd_dir which defines it."""
+    for submodule_path in execd_submodule_paths(command, execd_dir):
+        try:
+            subprocess.check_call(submodule_path, shell=True, stderr=stderr)
+        except subprocess.CalledProcessError as e:
+            hookenv.log("Error ({}) running  {}. Output: {}".format(
+                e.returncode, e.cmd, e.output))
+            if die_on_error:
+                sys.exit(e.returncode)
+
+
+def execd_preinstall(execd_dir=None):
+    """Run charm-pre-install for each module within execd_dir."""
+    execd_run('charm-pre-install', execd_dir=execd_dir)

hooks/swift_storage_hooks.py

@@ -25,6 +25,7 @@ from charmhelpers.core.hookenv import (
 
 from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.host import restart_on_change
+from charmhelpers.payload.execd import execd_preinstall
 
 from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
@@ -37,6 +38,7 @@ CONFIGS = register_configs()
 
 @hooks.hook()
 def install():
+    execd_preinstall()
     configure_installation_source(config('openstack-origin'))
     apt_update()
     apt_install(PACKAGES, fatal=True)

unit_tests/test_swift_storage_context.py

@@ -1,9 +1,7 @@
 from mock import MagicMock
+from test_utils import CharmTestCase, patch_open
 
-from unit_tests.test_utils import CharmTestCase, patch_open
-
-
-import hooks.swift_storage_context as swift_context
+import swift_storage_context as swift_context
 
 
 TO_PATCH = [

unit_tests/test_swift_storage_relations.py

@@ -1,17 +1,17 @@
 from mock import patch, MagicMock
 
-from unit_tests.test_utils import CharmTestCase
+from test_utils import CharmTestCase
 
-import hooks.swift_storage_utils as utils
+import swift_storage_utils as utils
 
 _reg = utils.register_configs
 utils.register_configs = MagicMock()
 
-import hooks.swift_storage_hooks as hooks
+import swift_storage_hooks as hooks
 
 utils.register_configs = _reg
 
-from hooks.swift_storage_utils import PACKAGES
+from swift_storage_utils import PACKAGES
 
 TO_PATCH = [
     'CONFIGS',
@@ -35,6 +35,7 @@ TO_PATCH = [
     'save_script_rc',
     'setup_storage',
     'register_configs',
+    'execd_preinstall'
 ]
 
 
@@ -55,6 +56,7 @@ class SwiftStorageRelationsTests(CharmTestCase):
         self.apt_install.assert_called_with(PACKAGES, fatal=True)
 
         self.setup_storage.assert_called()
+        self.execd_preinstall.assert_called()
 
     def test_config_changed_no_upgrade_available(self):
         self.openstack_upgrade_available.return_value = False

unit_tests/test_swift_storage_utils.py

@@ -1,8 +1,7 @@
 from mock import call, patch, MagicMock
-from unit_tests.test_utils import CharmTestCase, patch_open
+from test_utils import CharmTestCase, patch_open
 
-
-import hooks.swift_storage_utils as swift_utils
+import swift_storage_utils as swift_utils
 
 
 TO_PATCH = [
@@ -172,7 +171,7 @@ class SwiftStorageUtilsTests(CharmTestCase):
         swift_utils.save_script_rc()
         self._save_script_rc.assert_called_with(**SCRIPT_RC_ENV)
 
-    @patch('hooks.charmhelpers.contrib.openstack.templating.OSConfigRenderer')
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
     def test_register_configs_pre_install(self, renderer):
         self.get_os_codename_package.return_value = None
         swift_utils.register_configs()
@@ -182,7 +181,7 @@ class SwiftStorageUtilsTests(CharmTestCase):
     @patch.object(swift_utils, 'SwiftStorageContext')
     @patch.object(swift_utils, 'RsyncContext')
     @patch.object(swift_utils, 'SwiftStorageServerContext')
-    @patch('hooks.charmhelpers.contrib.openstack.templating.OSConfigRenderer')
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
     def test_register_configs_post_install(self, renderer,
                                            swift, rsync, server):
         swift.return_value = 'swift_context'