Sysinv api load import improvements
Disk availability check has been added to sysinv api request processing via pecan hook. Also, close of second webob temporary copy and proper error parsing on middleware have been added. As result of this change, the disk space required reduced from 3x to 2x the file size, also, the processing does not proceed if there is no space available. Test Plan: load-import directly on sysinv (System Controller) PASS: Verify if disk space available less than 2x file size PASS: Verify if disk space available higher than 2x file size PASS: Verify proper error returned when no space available PASS: Verify proper load import if there is space available PASS: Verify deleting and re-importing the load PASS: Verify disk space needed decreased 1x the file size PASS: Verify the temporary /scratch file copies are always removed Regression: load-import to several subclouds PASS: Verify load import to 50 subclouds, 2 in parallel PASS: Verify load import to 50 subclouds, all in parallel PASS: Verify the time to accomplish 50 subclouds is acceptable Story: 2009266 Task: 43501 Bug: 1945737 Signed-off-by: Adriano Oliveira <adriano.oliveira@windriver.com> Change-Id: Id5a04d0e39e42afd0578e521f79739ef3b360231
This commit is contained in:
parent
ce5d15b6f4
commit
06230585f9
|
@ -54,8 +54,8 @@ def make_tempdir():
|
|||
def setup_app(pecan_config=None, extra_hooks=None):
|
||||
policy.init()
|
||||
|
||||
# hooks.DBTransactionHook()
|
||||
app_hooks = [hooks.ConfigHook(),
|
||||
app_hooks = [hooks.MultiFormDataHook(),
|
||||
hooks.ConfigHook(),
|
||||
hooks.DBHook(),
|
||||
hooks.ContextHook(pecan_config.app.acl_public_routes),
|
||||
hooks.RPCHook(),
|
||||
|
|
|
@ -39,6 +39,7 @@ from sysinv.openstack.common import policy
|
|||
from webob import exc
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
NO_SPACE_MSG = "Insufficient space"
|
||||
|
||||
audit_log_name = "{}.{}".format(__name__, "auditor")
|
||||
auditLOG = log.getLogger(audit_log_name)
|
||||
|
@ -48,6 +49,46 @@ def generate_request_id():
|
|||
return 'req-%s' % uuidutils.generate_uuid()
|
||||
|
||||
|
||||
def is_load_import(content_type, url_path):
|
||||
if (content_type == "multipart/form-data" and
|
||||
url_path == "/v1/loads/import_load"):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
class MultiFormDataHook(hooks.PecanHook):
|
||||
"""For multipart form-data, check disk space available before
|
||||
proceeding.
|
||||
|
||||
Currently, it is only applying to import_load request, but
|
||||
it can be extended to cover other multipart form-data requests
|
||||
"""
|
||||
|
||||
def on_route(self, state):
|
||||
content_type = state.request.content_type
|
||||
url_path = state.request.path
|
||||
if is_load_import(content_type, url_path):
|
||||
content_length = int(state.request.headers.get('Content-Length'))
|
||||
# Currently, the restriction is 2x the file size:
|
||||
# 1x from internal webob copy (see before override below)
|
||||
# 1x from sysinv temporary copy
|
||||
if not utils.is_space_available("/scratch", 2 * content_length):
|
||||
msg = _(NO_SPACE_MSG + " on /scratch for request %s"
|
||||
% url_path)
|
||||
raise webob.exc.HTTPInternalServerError(explanation=msg)
|
||||
|
||||
# Note: webob, for the multipart form-data request, creates 2 internal
|
||||
# temporary copies, using the before override we can close the second
|
||||
# temporary request before request goes to sysinv, this saves 1x file
|
||||
# size required
|
||||
def before(self, state):
|
||||
content_type = state.request.content_type
|
||||
url_path = state.request.path
|
||||
if is_load_import(content_type, url_path):
|
||||
state.request.body_file.close()
|
||||
|
||||
|
||||
class ConfigHook(hooks.PecanHook):
|
||||
"""Attach the config object to the request so controllers can get to it."""
|
||||
|
||||
|
@ -230,7 +271,13 @@ class AuditLogging(hooks.PecanHook):
|
|||
|
||||
def json_post_data(rest_state):
|
||||
if 'form-data' in rest_state.request.headers.get('Content-Type'):
|
||||
return " POST: {}".format(rest_state.request.params)
|
||||
# rest_state.request.params causes an internal webob copy,
|
||||
# prevent its call if there is no space available
|
||||
size = int(rest_state.request.headers.get('Content-Length'))
|
||||
if utils.is_space_available("/scratch", 2 * size):
|
||||
return " POST: {}".format(rest_state.request.params)
|
||||
else:
|
||||
return " POST: " + NO_SPACE_MSG + " for processing"
|
||||
if not hasattr(rest_state.request, 'json'):
|
||||
return ""
|
||||
return " POST: {}".format(rest_state.request.json)
|
||||
|
|
|
@ -31,6 +31,12 @@ from oslo_log import log
|
|||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
# As per webob.exc code:
|
||||
# https://github.com/Pylons/webob/blob/master/src/webob/exc.py
|
||||
# The explanation field is added to the HTTP exception as following:
|
||||
# ${explanation}<br /><br />
|
||||
WEBOB_EXPL_SEP = "<br /><br />"
|
||||
|
||||
|
||||
class ParsableErrorMiddleware(object):
|
||||
"""Replace error body with something the client can parse.
|
||||
|
@ -86,7 +92,20 @@ class ParsableErrorMiddleware(object):
|
|||
else:
|
||||
if six.PY3:
|
||||
app_iter = [i.decode('utf-8') for i in app_iter]
|
||||
body = [json.dumps({'error_message': '\n'.join(app_iter)})]
|
||||
# Parse explanation field from webob.exc and add it as
|
||||
# 'faultstring' to be processed by cgts-client
|
||||
fault = None
|
||||
app_data = '\n'.join(app_iter)
|
||||
for data in app_data.split("\n"):
|
||||
if WEBOB_EXPL_SEP in str(data):
|
||||
# Remove separator, trailing and leading white spaces
|
||||
fault = str(data).replace(WEBOB_EXPL_SEP, "").strip()
|
||||
break
|
||||
if fault is None:
|
||||
body = [json.dumps({'error_message': app_data})]
|
||||
else:
|
||||
body = [json.dumps({'error_message':
|
||||
json.dumps({'faultstring': fault})})]
|
||||
if six.PY3:
|
||||
body = [item.encode('utf-8') for item in body]
|
||||
state['headers'].append(('Content-Type', 'application/json'))
|
||||
|
|
|
@ -47,6 +47,7 @@ import keyring
|
|||
import math
|
||||
import os
|
||||
import pathlib
|
||||
import psutil
|
||||
import pwd
|
||||
import random
|
||||
import re
|
||||
|
@ -1279,6 +1280,14 @@ def is_cpe(host_obj):
|
|||
host_has_function(host_obj, constants.WORKER))
|
||||
|
||||
|
||||
def is_space_available(partition, size):
|
||||
"""
|
||||
Returns if the given size is available in the specified partition
|
||||
"""
|
||||
available_space = psutil.disk_usage(partition).free
|
||||
return False if available_space < size else True
|
||||
|
||||
|
||||
def output_to_dict(output):
|
||||
dict = {}
|
||||
output = [_f for _f in output.split('\n') if _f]
|
||||
|
|
Loading…
Reference in New Issue