Implement in-memory data processing for upload hooks

This code adds a possibility to unpack zip archives
in ram for 'hooks_artifact' type.

Change-Id: I40417d96593f6545f205e51af40535151c2aea96

glare/tests/hooks_artifact.py

@@ -12,14 +12,17 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+import io
 import os
 import shutil
+import zipfile
 
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_versionedobjects import fields
 
 from glare.common import exception
+from glare.common import utils
 from glare.objects import base
 from glare.objects.meta import file_utils
 from glare.objects.meta import wrappers
@@ -46,15 +49,37 @@ class HookChecker(base.BaseArtifact):
         'forbid_download_zip': Field(fields.FlexibleBooleanField,
                                      default=False),
         'forbid_delete': Field(fields.FlexibleBooleanField,
-                               default=False, mutable=True)
+                               default=False, mutable=True),
     }
 
+    artifact_type_opts = base.BaseArtifact.artifact_type_opts + [
+        cfg.BoolOpt('in_memory_processing')
+    ]
+
     @classmethod
     def get_type_name(cls):
         return "hooks_artifact"
 
     @classmethod
-    def validate_upload(cls, context, af, field_name, fd):
+    def _validate_upload_inmemory(cls, context, af, field_name, fd):
+        flobj = io.BytesIO(fd.read())
+        while True:
+            data = fd.read(65536)
+            if data == b'':  # end of file reached
+                break
+            flobj.write(data)
+
+        zip_ref = zipfile.ZipFile(flobj, 'r')
+        for name in zip_ref.namelist():
+            if not name.endswith('/'):
+                file_utils.upload_content_file(
+                    context, af, utils.BlobIterator(zip_ref.read(name)),
+                    'content', name)
+        flobj.seek(0)
+        return flobj, None
+
+    @classmethod
+    def _validate_upload_harddrive(cls, context, af, field_name, fd):
         path = None
         tdir = None
         try:
@@ -83,6 +108,15 @@ class HookChecker(base.BaseArtifact):
         tfile.seek(0)
         return tfile, path
 
+    @classmethod
+    def validate_upload(cls, context, af, field_name, fd):
+        if CONF.hooks_artifact.in_memory_processing:
+            return cls._validate_upload_harddrive(
+                context, af, field_name, fd)
+        else:
+            return cls._validate_upload_inmemory(
+                context, af, field_name, fd)
+
     @classmethod
     def validate_download(cls, context, af, field_name, fd):
         if af.forbid_download_zip and field_name == 'zip':

glare/tests/unit/test_validation_hooks.py

@@ -44,6 +44,13 @@ class TestArtifactHooks(base.BaseTestArtifactAPI):
         self.assertEqual(
             11, artifact['content']['folder1/folder2/ccc.txt']['size'])
 
+    def test_upload_hook_inmemory(self):
+        # enable in-memory processing
+        self.config(in_memory_processing=True,
+                    group='hooks_artifact')
+
+        self.test_upload_hook()
+
     def test_download_hook(self):
         # upload data
         var_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),