Implement in-memory data processing for upload hooks
This code adds a possibility to unpack zip archives in ram for 'hooks_artifact' type. Change-Id: I40417d96593f6545f205e51af40535151c2aea96
This commit is contained in:
parent
5da4af347c
commit
4c0b88e8eb
glare/tests
@ -12,14 +12,17 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import zipfile
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
from oslo_versionedobjects import fields
|
||||
|
||||
from glare.common import exception
|
||||
from glare.common import utils
|
||||
from glare.objects import base
|
||||
from glare.objects.meta import file_utils
|
||||
from glare.objects.meta import wrappers
|
||||
@ -46,15 +49,37 @@ class HookChecker(base.BaseArtifact):
|
||||
'forbid_download_zip': Field(fields.FlexibleBooleanField,
|
||||
default=False),
|
||||
'forbid_delete': Field(fields.FlexibleBooleanField,
|
||||
default=False, mutable=True)
|
||||
default=False, mutable=True),
|
||||
}
|
||||
|
||||
artifact_type_opts = base.BaseArtifact.artifact_type_opts + [
|
||||
cfg.BoolOpt('in_memory_processing')
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def get_type_name(cls):
|
||||
return "hooks_artifact"
|
||||
|
||||
@classmethod
|
||||
def validate_upload(cls, context, af, field_name, fd):
|
||||
def _validate_upload_inmemory(cls, context, af, field_name, fd):
|
||||
flobj = io.BytesIO(fd.read())
|
||||
while True:
|
||||
data = fd.read(65536)
|
||||
if data == b'': # end of file reached
|
||||
break
|
||||
flobj.write(data)
|
||||
|
||||
zip_ref = zipfile.ZipFile(flobj, 'r')
|
||||
for name in zip_ref.namelist():
|
||||
if not name.endswith('/'):
|
||||
file_utils.upload_content_file(
|
||||
context, af, utils.BlobIterator(zip_ref.read(name)),
|
||||
'content', name)
|
||||
flobj.seek(0)
|
||||
return flobj, None
|
||||
|
||||
@classmethod
|
||||
def _validate_upload_harddrive(cls, context, af, field_name, fd):
|
||||
path = None
|
||||
tdir = None
|
||||
try:
|
||||
@ -83,6 +108,15 @@ class HookChecker(base.BaseArtifact):
|
||||
tfile.seek(0)
|
||||
return tfile, path
|
||||
|
||||
@classmethod
|
||||
def validate_upload(cls, context, af, field_name, fd):
|
||||
if CONF.hooks_artifact.in_memory_processing:
|
||||
return cls._validate_upload_harddrive(
|
||||
context, af, field_name, fd)
|
||||
else:
|
||||
return cls._validate_upload_inmemory(
|
||||
context, af, field_name, fd)
|
||||
|
||||
@classmethod
|
||||
def validate_download(cls, context, af, field_name, fd):
|
||||
if af.forbid_download_zip and field_name == 'zip':
|
||||
|
@ -44,6 +44,13 @@ class TestArtifactHooks(base.BaseTestArtifactAPI):
|
||||
self.assertEqual(
|
||||
11, artifact['content']['folder1/folder2/ccc.txt']['size'])
|
||||
|
||||
def test_upload_hook_inmemory(self):
|
||||
# enable in-memory processing
|
||||
self.config(in_memory_processing=True,
|
||||
group='hooks_artifact')
|
||||
|
||||
self.test_upload_hook()
|
||||
|
||||
def test_download_hook(self):
|
||||
# upload data
|
||||
var_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||
|
Loading…
x
Reference in New Issue
Block a user