Support function package md5

User can specify md5sum for the package when creating function that
Qinling could check. User can also check by herself when downloading
function package.

Change-Id: Ib3d37cd92bd2ed7018f5a4825a5323b2652948c9
Implements: blueprint qinling-function-package-md5

qinling/api/controllers/v1/function.py

@@ -168,6 +168,7 @@ class FunctionsController(rest.RestController):
         create_trust = True
         if source == constants.PACKAGE_FUNCTION:
             store = True
+            md5sum = values['code'].get('md5sum')
             data = kwargs['package'].file.read()
         elif source == constants.SWIFT_FUNCTION:
             swift_info = values['code'].get('swift', {})
@@ -190,12 +191,14 @@ class FunctionsController(rest.RestController):
             func_db = db_api.create_function(values)
 
             if store:
-                ctx = context.get_ctx()
-                self.storage_provider.store(
-                    ctx.projectid,
-                    func_db.id,
-                    data
-                )
+                try:
+                    ctx = context.get_ctx()
+                    self.storage_provider.store(ctx.projectid, func_db.id,
+                                                data, md5sum=md5sum)
+                except Exception as e:
+                    LOG.exception("Failed to store function package.")
+                    keystone_util.delete_trust(values['trust_id'])
+                    raise e
 
         pecan.response.status = 201
         return resources.Function.from_dict(func_db.to_dict()).to_dict()

qinling/storage/base.py

@@ -27,7 +27,7 @@ class PackageStorage(object):
     """PackageStorage interface."""
 
     @abc.abstractmethod
-    def store(self, project_id, funtion, data):
+    def store(self, project_id, funtion, data, **kwargs):
         raise NotImplementedError
 
     @abc.abstractmethod

qinling/storage/file_system.py

@@ -21,6 +21,7 @@ from oslo_utils import fileutils
 
 from qinling import exceptions as exc
 from qinling.storage import base
+from qinling.utils import common
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -32,12 +33,12 @@ class FileSystemStorage(base.PackageStorage):
     def __init__(self, *args, **kwargs):
         fileutils.ensure_tree(CONF.storage.file_system_dir)
 
-    def store(self, project_id, function, data):
+    def store(self, project_id, function, data, md5sum=None):
         """Store the function package data to local file system.
 
         :param project_id: Project ID.
         :param function: Function ID.
-        :param data: Package data.
+        :param data: Package file content.
         """
         LOG.debug(
             'Store package, function: %s, project: %s', function, project_id
@@ -48,6 +49,13 @@ class FileSystemStorage(base.PackageStorage):
 
         new_func_zip = os.path.join(project_path, '%s.zip.new' % function)
         func_zip = os.path.join(project_path, '%s.zip' % function)
+
+        # Check md5
+        md5_actual = common.md5(content=data)
+        if md5sum and md5_actual != md5sum:
+            raise exc.InputException("Package md5 mismatch.")
+
+        # Store package
         with open(new_func_zip, 'wb') as fd:
             fd.write(data)
 

qinling/utils/common.py

@@ -12,6 +12,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 import functools
+import hashlib
 import pdb
 import sys
 import warnings
@@ -122,3 +123,16 @@ class ForkedPdb(pdb.Pdb):
             pdb.Pdb.interaction(self, *args, **kwargs)
         finally:
             sys.stdin = _stdin
+
+
+def md5(file=None, content=None):
+    hash_md5 = hashlib.md5()
+
+    if file:
+        with open(file, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                hash_md5.update(chunk)
+    elif content:
+        hash_md5.update(content)
+
+    return hash_md5.hexdigest()

qinling_tempest_plugin/services/qinling_client.py

@@ -60,7 +60,8 @@ class QinlingClient(client_base.QinlingClientBase):
         """Create function.
 
         Tempest rest client doesn't support multipart upload, so use requests
-        lib instead.
+        lib instead. As a result, we can not use self.assertRaises function for
+        negative tests.
         """
         headers = {'X-Auth-Token': self.auth_provider.get_token()}
         req_body = {

qinling_tempest_plugin/tests/api/test_functions.py

@@ -18,6 +18,7 @@ from tempest.lib import exceptions
 import tenacity
 
 from qinling_tempest_plugin.tests import base
+from qinling_tempest_plugin.tests import utils
 
 
 class FunctionsTest(base.BaseQinlingTest):
@@ -33,7 +34,8 @@ class FunctionsTest(base.BaseQinlingTest):
     @decorators.idempotent_id('9c36ac64-9a44-4c44-9e44-241dcc6b0933')
     def test_crud_function(self):
         # Create function
-        function_id = self.create_function(self.python_zip_file)
+        md5sum = utils.md5(self.python_zip_file)
+        function_id = self.create_function(self.python_zip_file, md5sum=md5sum)
 
         # Get functions
         resp, body = self.client.get_resources('functions')
@@ -52,6 +54,20 @@ class FunctionsTest(base.BaseQinlingTest):
         resp = self.client.delete_resource('functions', function_id)
         self.assertEqual(204, resp.status)
 
+    @decorators.idempotent_id('1fec41cd-b753-4cad-90c5-c89d7e710317')
+    def test_create_function_md5mismatch(self):
+        fake_md5 = "e807f1fcf82d132f9bb018ca6738a19f"
+
+        with open(self.python_zip_file, 'rb') as package_data:
+            resp, body = self.client.create_function(
+                {"source": "package", "md5sum": fake_md5},
+                self.runtime_id,
+                name='test_create_function_md5mismatch',
+                package_data=package_data
+            )
+
+        self.assertEqual(400, resp.status_code)
+
     @decorators.idempotent_id('051f3106-df01-4fcd-a0a3-c81c99653163')
     def test_get_all_admin(self):
         # Create function by normal user

qinling_tempest_plugin/tests/base.py

@@ -116,7 +116,7 @@ class BaseQinlingTest(test.BaseTestCase):
         self.addCleanup(os.remove, python_zip_file)
         return python_zip_file
 
-    def create_function(self, package_path=None, image=False):
+    def create_function(self, package_path=None, image=False, md5sum=None):
         function_name = data_utils.rand_name(
             'function',
             prefix=self.name_prefix
@@ -125,11 +125,15 @@ class BaseQinlingTest(test.BaseTestCase):
         if not image:
             if not package_path:
                 package_path = self.create_package()
+
+            code = {"source": "package"}
+            if md5sum:
+                code.update({"md5sum": md5sum})
             base_name, _ = os.path.splitext(package_path)
             module_name = os.path.basename(base_name)
             with open(package_path, 'rb') as package_data:
                 resp, body = self.client.create_function(
-                    {"source": "package"},
+                    code,
                     self.runtime_id,
                     name=function_name,
                     package_data=package_data,

qinling_tempest_plugin/tests/utils.py

@@ -12,6 +12,8 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+import hashlib
+
 from kubernetes.client import api_client
 from kubernetes.client.apis import core_v1_api
 from kubernetes.client.apis import extensions_v1beta1_api
@@ -32,3 +34,16 @@ def get_k8s_clients(conf):
     }
 
     return clients
+
+
+def md5(file=None, content=None):
+    hash_md5 = hashlib.md5()
+
+    if file:
+        with open(file, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                hash_md5.update(chunk)
+    elif content:
+        hash_md5.update(content)
+
+    return hash_md5.hexdigest()