From 2fc87a1494a6109d074a80c79cc3ac2f371e7283 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Fri, 19 Jan 2018 15:38:57 +1300 Subject: [PATCH] Support function package md5 User can specify md5sum for the package when creating function that Qinling could check. User can also check by herself when downloading function package. Change-Id: Ib3d37cd92bd2ed7018f5a4825a5323b2652948c9 Implements: blueprint qinling-function-package-md5 --- qinling/api/controllers/v1/function.py | 15 +++++++++------ qinling/storage/base.py | 2 +- qinling/storage/file_system.py | 12 ++++++++++-- qinling/utils/common.py | 14 ++++++++++++++ .../services/qinling_client.py | 3 ++- .../tests/api/test_functions.py | 18 +++++++++++++++++- qinling_tempest_plugin/tests/base.py | 8 ++++++-- qinling_tempest_plugin/tests/utils.py | 15 +++++++++++++++ 8 files changed, 74 insertions(+), 13 deletions(-) diff --git a/qinling/api/controllers/v1/function.py b/qinling/api/controllers/v1/function.py index 536ac259..b0334ff8 100644 --- a/qinling/api/controllers/v1/function.py +++ b/qinling/api/controllers/v1/function.py @@ -168,6 +168,7 @@ class FunctionsController(rest.RestController): create_trust = True if source == constants.PACKAGE_FUNCTION: store = True + md5sum = values['code'].get('md5sum') data = kwargs['package'].file.read() elif source == constants.SWIFT_FUNCTION: swift_info = values['code'].get('swift', {}) @@ -190,12 +191,14 @@ class FunctionsController(rest.RestController): func_db = db_api.create_function(values) if store: - ctx = context.get_ctx() - self.storage_provider.store( - ctx.projectid, - func_db.id, - data - ) + try: + ctx = context.get_ctx() + self.storage_provider.store(ctx.projectid, func_db.id, + data, md5sum=md5sum) + except Exception as e: + LOG.exception("Failed to store function package.") + keystone_util.delete_trust(values['trust_id']) + raise e pecan.response.status = 201 return resources.Function.from_dict(func_db.to_dict()).to_dict() diff --git a/qinling/storage/base.py b/qinling/storage/base.py index 3ebfc378..ffe3dcbf 100644 --- a/qinling/storage/base.py +++ b/qinling/storage/base.py @@ -27,7 +27,7 @@ class PackageStorage(object): """PackageStorage interface.""" @abc.abstractmethod - def store(self, project_id, funtion, data): + def store(self, project_id, funtion, data, **kwargs): raise NotImplementedError @abc.abstractmethod diff --git a/qinling/storage/file_system.py b/qinling/storage/file_system.py index 270f0698..f4074b2d 100644 --- a/qinling/storage/file_system.py +++ b/qinling/storage/file_system.py @@ -21,6 +21,7 @@ from oslo_utils import fileutils from qinling import exceptions as exc from qinling.storage import base +from qinling.utils import common LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -32,12 +33,12 @@ class FileSystemStorage(base.PackageStorage): def __init__(self, *args, **kwargs): fileutils.ensure_tree(CONF.storage.file_system_dir) - def store(self, project_id, function, data): + def store(self, project_id, function, data, md5sum=None): """Store the function package data to local file system. :param project_id: Project ID. :param function: Function ID. - :param data: Package data. + :param data: Package file content. """ LOG.debug( 'Store package, function: %s, project: %s', function, project_id @@ -48,6 +49,13 @@ class FileSystemStorage(base.PackageStorage): new_func_zip = os.path.join(project_path, '%s.zip.new' % function) func_zip = os.path.join(project_path, '%s.zip' % function) + + # Check md5 + md5_actual = common.md5(content=data) + if md5sum and md5_actual != md5sum: + raise exc.InputException("Package md5 mismatch.") + + # Store package with open(new_func_zip, 'wb') as fd: fd.write(data) diff --git a/qinling/utils/common.py b/qinling/utils/common.py index 7b920ffa..2bc237a4 100644 --- a/qinling/utils/common.py +++ b/qinling/utils/common.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import functools +import hashlib import pdb import sys import warnings @@ -122,3 +123,16 @@ class ForkedPdb(pdb.Pdb): pdb.Pdb.interaction(self, *args, **kwargs) finally: sys.stdin = _stdin + + +def md5(file=None, content=None): + hash_md5 = hashlib.md5() + + if file: + with open(file, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + elif content: + hash_md5.update(content) + + return hash_md5.hexdigest() diff --git a/qinling_tempest_plugin/services/qinling_client.py b/qinling_tempest_plugin/services/qinling_client.py index 4a1716bd..bda79746 100644 --- a/qinling_tempest_plugin/services/qinling_client.py +++ b/qinling_tempest_plugin/services/qinling_client.py @@ -60,7 +60,8 @@ class QinlingClient(client_base.QinlingClientBase): """Create function. Tempest rest client doesn't support multipart upload, so use requests - lib instead. + lib instead. As a result, we can not use self.assertRaises function for + negative tests. """ headers = {'X-Auth-Token': self.auth_provider.get_token()} req_body = { diff --git a/qinling_tempest_plugin/tests/api/test_functions.py b/qinling_tempest_plugin/tests/api/test_functions.py index 02ce1d11..5959f455 100644 --- a/qinling_tempest_plugin/tests/api/test_functions.py +++ b/qinling_tempest_plugin/tests/api/test_functions.py @@ -18,6 +18,7 @@ from tempest.lib import exceptions import tenacity from qinling_tempest_plugin.tests import base +from qinling_tempest_plugin.tests import utils class FunctionsTest(base.BaseQinlingTest): @@ -33,7 +34,8 @@ class FunctionsTest(base.BaseQinlingTest): @decorators.idempotent_id('9c36ac64-9a44-4c44-9e44-241dcc6b0933') def test_crud_function(self): # Create function - function_id = self.create_function(self.python_zip_file) + md5sum = utils.md5(self.python_zip_file) + function_id = self.create_function(self.python_zip_file, md5sum=md5sum) # Get functions resp, body = self.client.get_resources('functions') @@ -52,6 +54,20 @@ class FunctionsTest(base.BaseQinlingTest): resp = self.client.delete_resource('functions', function_id) self.assertEqual(204, resp.status) + @decorators.idempotent_id('1fec41cd-b753-4cad-90c5-c89d7e710317') + def test_create_function_md5mismatch(self): + fake_md5 = "e807f1fcf82d132f9bb018ca6738a19f" + + with open(self.python_zip_file, 'rb') as package_data: + resp, body = self.client.create_function( + {"source": "package", "md5sum": fake_md5}, + self.runtime_id, + name='test_create_function_md5mismatch', + package_data=package_data + ) + + self.assertEqual(400, resp.status_code) + @decorators.idempotent_id('051f3106-df01-4fcd-a0a3-c81c99653163') def test_get_all_admin(self): # Create function by normal user diff --git a/qinling_tempest_plugin/tests/base.py b/qinling_tempest_plugin/tests/base.py index 2e61adff..278bcd55 100644 --- a/qinling_tempest_plugin/tests/base.py +++ b/qinling_tempest_plugin/tests/base.py @@ -116,7 +116,7 @@ class BaseQinlingTest(test.BaseTestCase): self.addCleanup(os.remove, python_zip_file) return python_zip_file - def create_function(self, package_path=None, image=False): + def create_function(self, package_path=None, image=False, md5sum=None): function_name = data_utils.rand_name( 'function', prefix=self.name_prefix @@ -125,11 +125,15 @@ class BaseQinlingTest(test.BaseTestCase): if not image: if not package_path: package_path = self.create_package() + + code = {"source": "package"} + if md5sum: + code.update({"md5sum": md5sum}) base_name, _ = os.path.splitext(package_path) module_name = os.path.basename(base_name) with open(package_path, 'rb') as package_data: resp, body = self.client.create_function( - {"source": "package"}, + code, self.runtime_id, name=function_name, package_data=package_data, diff --git a/qinling_tempest_plugin/tests/utils.py b/qinling_tempest_plugin/tests/utils.py index 54a87416..9fbe0d56 100644 --- a/qinling_tempest_plugin/tests/utils.py +++ b/qinling_tempest_plugin/tests/utils.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib + from kubernetes.client import api_client from kubernetes.client.apis import core_v1_api from kubernetes.client.apis import extensions_v1beta1_api @@ -32,3 +34,16 @@ def get_k8s_clients(conf): } return clients + + +def md5(file=None, content=None): + hash_md5 = hashlib.md5() + + if file: + with open(file, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + elif content: + hash_md5.update(content) + + return hash_md5.hexdigest()