Add unittests about validation method in storlet_docker_gateway

This patch adds unittests for the validation methods in
storlet_docker_gateway.StorletGatewayDocker

Bonus of this patch:
 * This patch improves checking logic about permission string
 * This patch makes validation methods to raise HTTPExceptions,
   whith makes it easier to handle responses.

Co-Authored-By: Kota Tsuyuzaki<tsuyuzaki.kota@lab.ntt.co.jp>
Change-Id: Ieb10ea20be3bc2be69b1eec2513d4e11e6ad48de

Engine/swift/storlet_gateway/storlet_docker_gateway.py

@@ -29,7 +29,7 @@ from storlet_runtime import RunTimePaths, RunTimeSandbox, \
     StorletInvocationGETProtocol, StorletInvocationPUTProtocol, \
     StorletInvocationSLOProtocol
 from swift.common.internal_client import InternalClient as ic
-from swift.common.swob import Request
+from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized
 from swift.common.utils import config_true_value
 
 
@@ -184,34 +184,30 @@ class StorletGatewayDocker(StorletGatewayBase):
         def __del__(self):
             self.close()
 
-    def _validateStorletUpload(self, req):
+    def _validate_storlet_upload(self, req):
         if (self.obj.find('-') < 0 or self.obj.find('.') < 0):
-            return 'Storlet name is incorrect'
+            raise HTTPBadRequest('Storlet name is incorrect', request=req)
 
-    def _validateDependencyUpload(self, req):
+    def _validate_dependency_upload(self, req):
         perm = req.headers. \
             get('X-Object-Meta-Storlet-Dependency-Permissions')
         if perm is not None:
             try:
-                perm_int = int(perm)
+                perm_int = int(perm, 8)
             except ValueError:
-                return 'Dependency permission is incorrect'
-            if perm_int / 100 < 6:
-                return 'The owner should have rw permission'
+                raise HTTPBadRequest('Dependency permission is incorrect',
+                                     request=req)
+            if (perm_int & int('600', 8)) != int('600', 8):
+                raise HTTPBadRequest('The owner hould have rw permission',
+                                     request=req)
 
     def validateStorletUpload(self, req):
-        ret = self._validate_mandatory_headers(req)
-        if ret:
-            return ret
+        self._validate_mandatory_headers(req)
 
         if (self.container == self.sconf['storlet_container']):
-            ret = self._validateStorletUpload(req)
+            self._validate_storlet_upload(req)
         elif (self.container == self.sconf['storlet_dependency']):
-            ret = self._validateDependencyUpload(req)
-        if ret:
-            return ret
-
-        return False
+            self._validate_dependency_upload(req)
 
     def authorizeStorletExecution(self, req):
         res, headers = self.verify_access(req.environ,
@@ -220,11 +216,11 @@ class StorletGatewayDocker(StorletGatewayBase):
                                           self.sconf['storlet_container'],
                                           req.headers['X-Run-Storlet'])
         if not res:
-            return False
+            raise HTTPUnauthorized('Account disabled for storlets',
+                                   request=req)
 
         # keep the storlets headers for later use.
         self.storlet_metadata = headers
-        return True
 
     def augmentStorletRequest(self, req):
         if self.storlet_metadata:
@@ -362,8 +358,8 @@ class StorletGatewayDocker(StorletGatewayBase):
                 if md not in req.headers:
                     self.logger.info('Mandatory header ' +
                                      'is missing: {0}'.format(md))
-                    return 'Mandatory header is missing: {0}'.format(md)
-        return None
+                    raise HTTPBadRequest('Mandatory header is missing'
+                                         ': {0}'.format(md))
 
     def _fix_request_headers(self, req):
         # add to request the storlet metadata to be used in case the request

Engine/swift/storlet_middleware/storlet_handler.py

@@ -24,8 +24,7 @@ from eventlet import Timeout
 from storlet_common import StorletException, StorletTimeout
 from swift.common.exceptions import ConnectionTimeout
 from swift.common.swob import HTTPBadRequest, HTTPException, \
-    HTTPInternalServerError, HTTPNotFound, HTTPUnauthorized, Request, \
-    Response, wsgify
+    HTTPInternalServerError, HTTPNotFound, Request, Response, wsgify
 from swift.common.utils import config_true_value, get_logger, is_success, \
     register_swift_info
 from swift.proxy.controllers.base import get_account_info
@@ -55,7 +54,13 @@ class StorletHandlerMiddleware(object):
                 device, partition, account, container, obj = \
                     req.split_path(5, 5, rest_with_last=True)
                 version = '0'
-        except Exception as e:
+        except ValueError:
+            # TODO(kajinamit): Can we merge the following checking
+            #                  about the target here?
+            return req.get_response(self.app)
+
+        # The target should be object
+        if not (account and container and obj):
             return req.get_response(self.app)
 
         self.logger.debug('storlet_handler call in %s: with %s/%s/%s' %
@@ -64,8 +69,8 @@ class StorletHandlerMiddleware(object):
         storlet_execution = False
         if 'X-Run-Storlet' in req.headers:
             storlet_execution = True
-        if (storlet_execution is True and account and container and obj) or \
-                (container in self.storlet_containers and obj):
+
+        if storlet_execution or container in self.storlet_containers:
             gateway = self.gateway_module(self.gateway_conf,
                                           self.logger, self.app, version,
                                           account, container, obj)
@@ -123,11 +128,11 @@ class StorletHandlerMiddleware(object):
                                                         'False')
                     if not config_true_value(storlets_enabled):
                         self.logger.info('Account disabled for storlets')
-                        return HTTPBadRequest('Account disabled for storlets')
+                        raise HTTPBadRequest('Account disabled for storlets',
+                                             request=req)
 
                 if req.method == 'GET' and storlet_execution:
-                    if not gateway.authorizeStorletExecution(req):
-                        return HTTPUnauthorized('Storlet: no permission')
+                    gateway.authorizeStorletExecution(req)
 
                     # The get request may be a SLO object GET request.
                     # Simplest solution would be to invoke a HEAD
@@ -192,12 +197,9 @@ class StorletHandlerMiddleware(object):
 
                 elif req.method == 'PUT':
                     if (container in self.storlet_containers):
-                        ret = gateway.validateStorletUpload(req)
-                        if ret:
-                            return HTTPBadRequest(body=ret)
+                        gateway.validateStorletUpload(req)
                     else:
-                        if not gateway.authorizeStorletExecution(req):
-                            return HTTPUnauthorized('Storlet: no permissions')
+                        gateway.authorizeStorletExecution(req)
                     if storlet_execution:
                         gateway.augmentStorletRequest(req)
                         (out_md, app_iter) = \
@@ -210,13 +212,13 @@ class StorletHandlerMiddleware(object):
 
         except (StorletTimeout, ConnectionTimeout, Timeout) as e:
             StorletException.handle(self.logger, e)
-            return HTTPInternalServerError(body='Storlet execution timed out')
+            raise HTTPInternalServerError(body='Storlet execution timed out')
         except HTTPException as e:
             StorletException.handle(self.logger, e)
-            return e
+            raise
         except Exception as e:
             StorletException.handle(self.logger, e)
-            return HTTPInternalServerError(body='Storlet execution failed')
+            raise HTTPInternalServerError(body='Storlet execution failed')
 
         return req.get_response(self.app)
 

SBusPythonFacade

@@ -0,0 +1 @@
+Engine/SBus/SBusPythonFacade/

setup.cfg

@@ -24,6 +24,7 @@ classifier =
 packages =
     storlet_middleware
     storlet_gateway
+    SBusPythonFacade
 
 [build_sphinx]
 source-dir = doc/source

tests/unit/swift/storlet_gateway/test_storlet_docker_gateway.py

@@ -0,0 +1,188 @@
+# Copyright (c) 2010-2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from storlet_gateway.storlet_docker_gateway import StorletGatewayDocker
+from swift.common.swob import HTTPException, Request, Response
+import unittest
+
+
+class FakeApp(object):
+    def __call__(self, env, start_response):
+        req = Request(env)
+        return Response(request=req, body='FAKE APP')(
+            env, start_response)
+
+
+class TestStorletGatewayDocker(unittest.TestCase):
+
+    # TODO(kajinamit): We have to implement FakeLogger
+    class FakeLogger(object):
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def debug(self, msg):
+            pass
+
+        def info(self, msg):
+            pass
+
+        def warn(self, msg):
+            pass
+
+        def error(self, msg):
+            pass
+
+    def setUp(self):
+        self.sconf = {
+            'lxc_root': '/home/docker_device/scopes',
+            'cache_dir': '/home/docker_device/cache/scopes',
+            'log_dir': '/home/docker_device/logs/scopes',
+            'script_dir': '/home/docker_device/scripts',
+            'storlets_dir': '/home/docker_device/storlets/scopes',
+            'pipes_dir': '/home/docker_device/pipes/scopes',
+            'storlet_timeout': '9',
+            'storlet_container': 'storlet',
+            'storlet_dependency': 'dependency'
+        }
+        self.logger = self.FakeLogger()
+
+    def tearDown(self):
+        pass
+
+    def assertRaisesWithStatus(self, status, f, *args, **kwargs):
+        try:
+            f(*args, **kwargs)
+        except HTTPException as e:
+            self.assertEqual(e.status_int, status)
+        else:
+            self.fail("HTTPException is not raised")
+
+    def test_validate_mandatory_headers_for_storlet(self):
+        a = 'AUTH_xxxxxxxxxxxxxxxxxxxx'
+        c = 'storlet'
+        o = 'storlet-1.0.jar'
+        path = '/'.join([a, c, o])
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+
+        # sufficient headers
+        headers = {'x-object-meta-storlet-language': 'java',
+                   'x-object-meta-storlet-interface-version': '1.0',
+                   'x-object-meta-storlet-dependency': 'dep_file',
+                   'x-object-meta-storlet-object-metadata': 'no',
+                   'x-object-meta-storlet-main': 'path.to.storlet.class'}
+        req = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'},
+                            headers=headers)
+        gw._validate_mandatory_headers(req)
+
+        # insufficient headers
+        headers = {'x-object-meta-storlet-language': 'java',
+                   'x-object-meta-storlet-interface-version': '1.0',
+                   'x-object-meta-storlet-dependency': 'dep_file',
+                   'x-object-meta-storlet-object-metadata': 'no'}
+        req = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'},
+                            headers=headers)
+        self.assertRaisesWithStatus(400, gw._validate_mandatory_headers, req)
+
+    def test_validate_mandatory_headers_for_dependency(self):
+        a = 'AUTH_xxxxxxxxxxxxxxxxxxxx'
+        c = 'dependency'
+        o = 'dep_file'
+        path = '/'.join([a, c, o])
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+
+        # sufficient headers
+        headers = {'x-object-meta-storlet-dependency-version': '1.0'}
+        req = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'},
+                            headers=headers)
+        gw._validate_mandatory_headers(req)
+
+        # insufficient headers
+        headers = {}
+        req = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'},
+                            headers=headers)
+        self.assertRaisesWithStatus(400, gw._validate_mandatory_headers, req)
+
+    def test_validate_storlet_upload(self):
+        a = 'AUTH_xxxxxxxxxxxxxxxxxxxx'
+        c = 'storlet'
+
+        # correct name
+        o = 'storlet-1.0.jar'
+        path = '/'.join([a, c, o])
+        req = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'})
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+        gw._validate_storlet_upload(req)
+
+        # wrong name
+        o = 'storlet.jar'
+        path = '/'.join([a, c, o])
+        req = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'})
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+        self.assertRaisesWithStatus(400, gw._validate_storlet_upload, req)
+
+    def test_validate_dependency_upload(self):
+        a = 'AUTH_xxxxxxxxxxxxxxxxxxxx'
+        c = 'dependency'
+        o = 'dep_file'
+        path = '/'.join([a, c, o])
+
+        # w/o dependency parameter
+        req = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'})
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+        gw._validate_dependency_upload(req)
+
+        # w/ correct dependency parameter
+        req = Request.blank(
+            path,
+            environ={'REQUEST_METHOD': 'PUT'},
+            headers={'x-object-meta-storlet-dependency-permissions': '755'})
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+        gw._validate_dependency_upload(req)
+
+        # w/ wrong dependency parameter
+        req = Request.blank(
+            path,
+            environ={'REQUEST_METHOD': 'PUT'},
+            headers={'x-object-meta-storlet-dependency-permissions': '400'})
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+        self.assertRaisesWithStatus(400, gw._validate_dependency_upload, req)
+
+        # w/ invalid dependency parameter
+        req = Request.blank(
+            path,
+            environ={'REQUEST_METHOD': 'PUT'},
+            headers={'x-object-meta-storlet-dependency-permissions': 'foo'})
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+        self.assertRaisesWithStatus(400, gw._validate_dependency_upload, req)
+
+        req = Request.blank(
+            path,
+            environ={'REQUEST_METHOD': 'PUT'},
+            headers={'x-object-meta-storlet-dependency-permissions': '888'})
+        gw = StorletGatewayDocker(self.sconf, self.logger, FakeApp(), 1.0,
+                                  a, c, o)
+        self.assertRaisesWithStatus(400, gw._validate_dependency_upload, req)
+
+
+if __name__ == '__main__':
+    unittest.main()