Merge "Run upload hooks in separate threads"

glare/engine.py

@@ -15,6 +15,7 @@
 
 from copy import deepcopy
 
+from eventlet import tpool
 import jsonpatch
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -574,10 +575,11 @@ class Engine(object):
                 if hasattr(af, 'validate_upload'):
                     LOG.warning("Method 'validate_upload' was deprecated. "
                                 "Please use 'pre_upload_hook' instead.")
-                    fd, path = af.validate_upload(context, af, field_name, fd)
+                    fd, path = tpool.execute(
+                        af.validate_upload, context, af, field_name, fd)
                 else:
-                    fd = af.pre_upload_hook(
-                        context, af, field_name, blob_key, fd)
+                    fd = tpool.execute(af.pre_upload_hook,
+                                       context, af, field_name, blob_key, fd)
             except exception.GlareException:
                 raise
             except Exception as e:

glare/tests/unit/base.py

@@ -79,12 +79,13 @@ class BaseTestCase(testtools.TestCase):
         self.config(
             custom_artifact_types_modules=[
                 'glare.tests.sample_artifact',
-                'glare.tests.hooks_artifact'
+                'glare.tests.hooks_artifact',
+                'glare.tests.unpacking_artifact'
             ],
             enabled_artifact_types=[
-                'hooks_artifact', 'sample_artifact', 'images',
-                'heat_templates', 'heat_environments', 'murano_packages',
-                'tosca_templates']
+                'unpacking_artifact', 'hooks_artifact', 'sample_artifact',
+                'images', 'heat_templates', 'heat_environments',
+                'murano_packages', 'tosca_templates']
         )
 
         location.SCHEME_TO_CLS_MAP = {}

glare/tests/unit/test_multistore.py

@@ -26,7 +26,8 @@ class TestMultistore(base.BaseTestCase):
                  'tosca_templates': 'sheepdog',
                  'murano_packages': 'vsphere',
                  'sample_artifact': 'database',
-                 'hooks_artifact': 'database'}
+                 'hooks_artifact': 'database',
+                 'unpacking_artifact': 'database'}
 
         # create engine and register new artifact types
         engine.Engine()

glare/tests/unit/test_unpacking.py

@@ -0,0 +1,44 @@
+# Copyright 2017 - Nokia Networks
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+
+from glare.tests.unit import base
+
+
+class TestArtifactHooks(base.BaseTestArtifactAPI):
+
+    def setUp(self):
+        super(TestArtifactHooks, self).setUp()
+        values = {'name': 'ttt', 'version': '1.0'}
+        self.unpacking_artifact = self.controller.create(
+            self.req, 'unpacking_artifact', values)
+
+    def test_unpacking(self):
+        var_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                               '../', 'var'))
+        data_path = os.path.join(var_dir, 'hooks.zip')
+        with open(data_path, "rb") as data:
+            self.controller.upload_blob(
+                self.req, 'unpacking_artifact', self.unpacking_artifact['id'],
+                'zip', data, 'application/octet-stream')
+        artifact = self.controller.show(self.req, 'unpacking_artifact',
+                                        self.unpacking_artifact['id'])
+        self.assertEqual(818, artifact['zip']['size'])
+        self.assertEqual('active', artifact['zip']['status'])
+
+        self.assertEqual(11, artifact['content']['aaa.txt']['size'])
+        self.assertEqual(11, artifact['content']['folder1/bbb.txt']['size'])
+        self.assertEqual(
+            11, artifact['content']['folder1/folder2/ccc.txt']['size'])

glare/tests/unpacking_artifact.py

@@ -0,0 +1,58 @@
+# Copyright 2017 - Nokia Networks
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import io
+import zipfile
+
+from glare.common import exception
+from glare.common import utils
+from glare.objects import base
+from glare.objects.meta import file_utils
+from glare.objects.meta import wrappers
+
+Blob = wrappers.BlobField.init
+Folder = wrappers.FolderField.init
+
+
+class Unpacker(base.BaseArtifact):
+    MAX_BLOB_SIZE = 100000
+
+    fields = {
+        'zip': Blob(description="Original zipped data.",
+                    required_on_activate=False),
+        'content': Folder(system=True, required_on_activate=False),
+    }
+
+    @classmethod
+    def get_type_name(cls):
+        return "unpacking_artifact"
+
+    @classmethod
+    def pre_upload_hook(cls, context, af, field_name, blob_key, fd):
+        flobj = io.BytesIO(fd.read(cls.MAX_BLOB_SIZE))
+
+        # Raise exception if something left in the stream
+        if fd.read(1):
+            msg = ("The file you are trying to upload is too big. "
+                   "The system upper limit is %s.") % cls.MAX_BLOB_SIZE
+            raise exception.RequestEntityTooLarge(msg)
+
+        zip_ref = zipfile.ZipFile(flobj, 'r')
+        for name in zip_ref.namelist():
+            if not name.endswith('/'):
+                file_utils.upload_content_file(
+                    context, af, utils.BlobIterator(zip_ref.read(name)),
+                    'content', name)
+        flobj.seek(0)
+        return flobj