diff --git a/glare/engine.py b/glare/engine.py index 19b920e..6b61c94 100644 --- a/glare/engine.py +++ b/glare/engine.py @@ -471,6 +471,8 @@ class Engine(object): try: # call upload hook first fd, path = af.validate_upload(context, af, field_name, fd) + except exception.GlareException: + raise except Exception as e: raise exception.BadRequest(message=str(e)) @@ -569,6 +571,8 @@ class Engine(object): # call download hook in the end data, path = af.validate_download( context, af, field_name, data) + except exception.GlareException: + raise except Exception as e: raise exception.BadRequest(message=str(e)) finally: diff --git a/glare/objects/meta/file_utils.py b/glare/objects/meta/file_utils.py index 9d2633d..738877a 100644 --- a/glare/objects/meta/file_utils.py +++ b/glare/objects/meta/file_utils.py @@ -25,13 +25,17 @@ from oslo_log import log as logging from oslo_utils import excutils from oslo_utils import uuidutils +from glare.common import exception from glare.common import store_api from glare.common import utils +from glare.i18n import _ from glare.objects.meta import fields as glare_fields CONF = cfg.CONF LOG = logging.getLogger(__name__) +INMEMORY_OBJECT_SIZE_LIMIT = 134217728 # 128 megabytes + def create_temporary_file(stream, suffix=''): """Create a temporary local file from a stream. @@ -111,17 +115,14 @@ def unpack_zip_archive_in_memory(context, af, field_name, fd): :param fd: zip archive :return io.BytesIO object - simple stream of in-memory bytes """ - # Warning: usage of this function is potentially harmful, because it - # doesn't limit how much data it writes to ram. Careless usage in artifact - # types may cause denial of the service. - # Thus it should be used only with blobs with reduced max_blob_size + flobj = io.BytesIO(fd.read(INMEMORY_OBJECT_SIZE_LIMIT)) - flobj = io.BytesIO(fd.read()) - while True: - data = fd.read(65536) - if data == b'': # end of file reached - break - flobj.write(data) + # Raise exception if something left + data = fd.read(1) + if data: + msg = _("The zip you are trying to unpack is too big. " + "The system upper limit is %s") % INMEMORY_OBJECT_SIZE_LIMIT + raise exception.RequestEntityTooLarge(msg) zip_ref = zipfile.ZipFile(flobj, 'r') for name in zip_ref.namelist(): diff --git a/glare/tests/unit/test_validation_hooks.py b/glare/tests/unit/test_validation_hooks.py index 84af163..18883f7 100644 --- a/glare/tests/unit/test_validation_hooks.py +++ b/glare/tests/unit/test_validation_hooks.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mock import os from glare.common import exception as exc @@ -49,6 +50,12 @@ class TestArtifactHooks(base.BaseTestArtifactAPI): self.config(in_memory_processing=True, group='hooks_artifact') + # First check uploading with smaller limit fails + with mock.patch('glare.objects.meta.file_utils.' + 'INMEMORY_OBJECT_SIZE_LIMIT', 817): + self.assertRaises(exc.RequestEntityTooLarge, self.test_upload_hook) + + # Now try with standard limit self.test_upload_hook() def test_download_hook(self):