Limit Ram Usage in unpack_zip_archive_in_memory.

Co-authored-by: Mike Fedosin <mikhail.fedosin.ext@nokia.com>

Change-Id: Ib93d539584bcc1b7f3f1095ccd2c8f1d74113db3

glare/engine.py

@@ -471,6 +471,8 @@ class Engine(object):
             try:
                 # call upload hook first
                 fd, path = af.validate_upload(context, af, field_name, fd)
+            except exception.GlareException:
+                raise
             except Exception as e:
                 raise exception.BadRequest(message=str(e))
 
@@ -569,6 +571,8 @@ class Engine(object):
             # call download hook in the end
             data, path = af.validate_download(
                 context, af, field_name, data)
+        except exception.GlareException:
+            raise
         except Exception as e:
             raise exception.BadRequest(message=str(e))
         finally:

glare/objects/meta/file_utils.py

@@ -25,13 +25,17 @@ from oslo_log import log as logging
 from oslo_utils import excutils
 from oslo_utils import uuidutils
 
+from glare.common import exception
 from glare.common import store_api
 from glare.common import utils
+from glare.i18n import _
 from glare.objects.meta import fields as glare_fields
 
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
+INMEMORY_OBJECT_SIZE_LIMIT = 134217728  # 128 megabytes
+
 
 def create_temporary_file(stream, suffix=''):
     """Create a temporary local file from a stream.
@@ -111,17 +115,14 @@ def unpack_zip_archive_in_memory(context, af, field_name, fd):
     :param fd: zip archive
     :return io.BytesIO object - simple stream of in-memory bytes
     """
-    #  Warning: usage of this function is potentially harmful, because it
-    #  doesn't limit how much data it writes to ram. Careless usage in artifact
-    #  types may cause denial of the service.
-    #  Thus it should be used only with blobs with reduced max_blob_size
+    flobj = io.BytesIO(fd.read(INMEMORY_OBJECT_SIZE_LIMIT))
 
-    flobj = io.BytesIO(fd.read())
-    while True:
-        data = fd.read(65536)
-        if data == b'':  # end of file reached
-            break
-        flobj.write(data)
+    # Raise exception if something left
+    data = fd.read(1)
+    if data:
+        msg = _("The zip you are trying to unpack is too big. "
+                "The system upper limit is %s") % INMEMORY_OBJECT_SIZE_LIMIT
+        raise exception.RequestEntityTooLarge(msg)
 
     zip_ref = zipfile.ZipFile(flobj, 'r')
     for name in zip_ref.namelist():

glare/tests/unit/test_validation_hooks.py

@@ -12,6 +12,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+import mock
 import os
 
 from glare.common import exception as exc
@@ -49,6 +50,12 @@ class TestArtifactHooks(base.BaseTestArtifactAPI):
         self.config(in_memory_processing=True,
                     group='hooks_artifact')
 
+        # First check uploading with smaller limit fails
+        with mock.patch('glare.objects.meta.file_utils.'
+                        'INMEMORY_OBJECT_SIZE_LIMIT', 817):
+            self.assertRaises(exc.RequestEntityTooLarge, self.test_upload_hook)
+
+        # Now try with standard limit
         self.test_upload_hook()
 
     def test_download_hook(self):