Add entry points for each gateway classes

This patch adds entry points for each gateway classes to enable
users to use these entry point name in the gateway congifration,
while we also can use the full class path for the gatewaay class.

For example, we can configure
 gateway_class = docker
or
 gateway_class = storlet_gateway.gateways.docker.StorletGatewayDocker
to load docker gateway.

Change-Id: Iffbd55bd46bb8259a91ecb261188e78088a34811

Engine/swift/etc/proxy-server.conf-sample

@@ -630,6 +630,6 @@ use = egg:swift#xprofile
 # storlet_container = storlet
 # storlet_dependency = dependency
 # storlet_timeout = 40
-# storlet_gateway_module = storlet_gateway.storlet_docker_gateway:StorletGatewayDocker
+# storlet_gateway_module = docker
 # storlet_gateway_conf = /etc/swift/storlet_stub_gateway.conf
 # execution_server = proxy

Engine/swift/setup.py

@@ -13,11 +13,18 @@ See the License for the specific language governing permissions and
 Limitations under the License.
 -------------------------------------------------------------------------"""
 from setuptools import setup, find_packages
+
+
 paste_factory = ['storlet_handler = '
                  'storlet_middleware.storlet_handler:filter_factory']
+gateways = [
+    'stub = storlet_gateway.gateways.stub:StorletGatewayStub',
+    'docker = storlet_gateway.gateways.docker:StorletGatewayDocker']
+
 
 setup(name='storlets',
       version='1.0',
       packages=find_packages(),
-      entry_points={'paste.filter_factory': paste_factory}
+      entry_points={'paste.filter_factory': paste_factory,
+                    'storlets.gateways': gateways}
       )

Engine/swift/storlet_gateway/loader.py

@@ -12,19 +12,38 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import sys
+from stevedore import driver
 from storlet_gateway.common.exceptions import StorletGatewayLoadError
 
 
-def load_gateway(module_name):
+def load_gateway(gateway_name):
-    module_name_sp = module_name.split(':')
+    """
-    if not len(module_name_sp):
+    Load gateway class for the given name
-        raise StorletGatewayLoadError('Invalid module name: %s' %
-                                      module_name)
 
+    :param gateway_name: a name of the gateway class to be loaded
+                         you can specify its entry point name or full path
+    :returns: a gateway class loaded
+    :raises StorletGatewayLoadError: when it fails to load the given gateway
+    """
+
+    namespace = 'storlets.gateways'
     try:
-        # TODO(takashi): Can we use stevedore to load module?
+        # Try to load gateway class using entry point
-        module = __import__(module_name_sp[0], fromlist=[module_name_sp[1]])
+        driver_manager = driver.DriverManager(namespace, gateway_name,
-        return getattr(module, module_name_sp[1])
+                                              invoke_on_load=False)
-    except (ImportError, AttributeError) as e:
+        return driver_manager.driver
-        raise StorletGatewayLoadError('Failed to load gateway class: %s' %
+    except RuntimeError:
-                                      str(e))
+        # Currently we can not find the entry point, so try again to load
+        # using full class path
+        mod_str, _sep, class_str = gateway_name.rpartition('.')
+        if not mod_str:
+            raise StorletGatewayLoadError(
+                'Invalid class path is given for gateway class name %s' %
+                gateway_name)
+        try:
+            __import__(mod_str)
+            return getattr(sys.modules[mod_str], class_str)
+        except (ImportError, AttributeError):
+            raise StorletGatewayLoadError('Failed to load gateway %s' %
+                                          gateway_name)

Engine/swift/storlet_middleware/storlet_handler.py

@@ -102,8 +102,7 @@ def filter_factory(global_conf, **local_conf):
         config_true_value(conf.get('storlet_execute_on_proxy_only', 'false'))
     storlet_conf['reseller_prefix'] = conf.get('reseller_prefix', 'AUTH')
 
-    module_name = conf.get('storlet_gateway_module',
+    module_name = conf.get('storlet_gateway_module', 'stub')
-                           'storlet_gateway.gateways.stub:StorletGatewayStub')
     gateway_class = load_gateway(module_name)
 
     configParser = ConfigParser.RawConfigParser()

doc/source/deployer_installation.rst

@@ -192,7 +192,7 @@ that do not appear in the above configurable installation_vars.yml
           "storlet_log": "storletlog",
           "storlet_images": "docker_images",
           "storlet_timeout": "40",
-          "storlet_gateway_module": "storlet_gateway.storlet_docker_gateway:StorletGatewayDocker",
+          "storlet_gateway_module": "docker",
           "storlet_execute_on_proxy_only": "false",
           "restart_linux_container_timeout": "3"
       }

doc/source/engine_dev_installation.rst

@@ -407,7 +407,7 @@ do the following:
     storlet_container = storlet
     storlet_dependency = dependency
     storlet_timeout = 40
-    storlet_gateway_module = storlet_gateway.storlet_docker_gateway:StorletGatewayDocker
+    storlet_gateway_module = docker
     storlet_gateway_conf = /etc/swift/storlet_docker_gateway.conf
     storlet_execute_on_proxy_only = false
     execution_server = proxy
@@ -432,7 +432,7 @@ otherwise the file is typically /etc/swift/object-server.conf
     storlet_container = storlet
     storlet_dependency = dependency
     storlet_timeout = 40
-    storlet_gateway_module = storlet_gateway.storlet_docker_gateway:StorletGatewayDocker
+    storlet_gateway_module = docker
     storlet_gateway_conf = /etc/swift/storlet_docker_gateway.conf
     storlet_execute_on_proxy_only = false
     execution_server = object

install/storlets/templates/cluster_config

@@ -45,7 +45,7 @@
         "storlet_log": "storletlog",
         "storlet_images": "docker_images",
         "storlet_timeout": "40",
-        "storlet_gateway_module": "storlet_gateway.gateways.docker:StorletGatewayDocker",
+        "storlet_gateway_module": "docker",
         "storlet_execute_on_proxy_only": "false",
         "restart_linux_container_timeout": "3"
     }

requirements.txt

@@ -8,3 +8,4 @@ Babel>=1.3
 eventlet>=0.17.4 # MIT
 greenlet>=0.3.1
 oslosphinx>=2.2.0  # Apache-2.0
+stevedore>=1.16.0  # Apache-2.0

setup.cfg

@@ -28,6 +28,10 @@ packages =
 paste.filter_factory =
     storlet_handler = storlet_middleware.storlet_handler:filter_factory
 
+storlets.gateways =
+    stub = storlet_gateway.gateways.stub:StorletGatewayStub
+    docker = storlet_gateway.gateways.docker:StorletGatewayDocker
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build

tests/unit/swift/storlet_gateway/test_loader.py

@@ -16,19 +16,44 @@ import unittest
 from storlet_gateway.common.exceptions import StorletGatewayLoadError
 from storlet_gateway.loader import load_gateway
 from storlet_gateway.gateways.stub import StorletGatewayStub
+from storlet_gateway.gateways.docker import StorletGatewayDocker
 
 
 class TestLoader(unittest.TestCase):
     def setUp(self):
         pass
 
-    def test_load_gateway(self):
+    def test_load_gateway_entry_point(self):
+        # existing entry point
         self.assertEqual(
             StorletGatewayStub,
-            load_gateway('storlet_gateway.gateways.stub:StorletGatewayStub'))
+            load_gateway('stub'))
 
-        with self.assertRaises(StorletGatewayLoadError):
+        self.assertEqual(
-            load_gateway('storlet_gateway.gateways.foo:StorletGatewayFoo')
+            StorletGatewayDocker,
+            load_gateway('docker'))
 
+        # If the given entry point does not exist
         with self.assertRaises(StorletGatewayLoadError):
-            load_gateway('storlet_gateway.gateways.stub:StorletGatewayFoo')
+            load_gateway('foo')
+
+    def test_load_gateway_full_class_path(self):
+        # If the given class path exists
+        self.assertEqual(
+            StorletGatewayStub,
+            load_gateway('storlet_gateway.gateways.stub.StorletGatewayStub'))
+
+        self.assertEqual(
+            StorletGatewayDocker,
+            load_gateway('storlet_gateway.gateways.docker.'
+                         'StorletGatewayDocker'))
+
+        # If module does not exist
+        with self.assertRaises(StorletGatewayLoadError):
+            load_gateway('storlet_gateway.gateways.another_stub.'
+                         'StorletGatewayStub')
+
+        # If class does not exist
+        with self.assertRaises(StorletGatewayLoadError):
+            load_gateway('storlet_gateway.gateways.stub.'
+                         'StorletGatewayAnotherStub')

tests/unit/swift/storlet_middleware/handlers/__init__.py

@@ -29,8 +29,7 @@ DEFAULT_CONFIG = {
     'storlet_container': 'storlet',
     'storlet_dependency': 'dependency',
     'storlet_timeout': '40',
-    'storlet_gateway_module':
+    'storlet_gateway_module': 'stub',
-        'storlet_gateway.gateways.stub:StorletGatewayStub',
     'storlet_gateway_conf': '/etc/swift/storlet_stub_gateway.conf',
     'execution_server': 'proxy'}