Merge "Sysinv api load import improvements"

sysinv/sysinv/sysinv/sysinv/api/app.py

@@ -54,8 +54,8 @@ def make_tempdir():
 def setup_app(pecan_config=None, extra_hooks=None):
     policy.init()
 
-    #            hooks.DBTransactionHook()
-    app_hooks = [hooks.ConfigHook(),
+    app_hooks = [hooks.MultiFormDataHook(),
+                 hooks.ConfigHook(),
                  hooks.DBHook(),
                  hooks.ContextHook(pecan_config.app.acl_public_routes),
                  hooks.RPCHook(),

sysinv/sysinv/sysinv/sysinv/api/hooks.py

@@ -39,6 +39,7 @@ from sysinv.openstack.common import policy
 from webob import exc
 
 LOG = log.getLogger(__name__)
+NO_SPACE_MSG = "Insufficient space"
 
 audit_log_name = "{}.{}".format(__name__, "auditor")
 auditLOG = log.getLogger(audit_log_name)
@@ -48,6 +49,46 @@ def generate_request_id():
     return 'req-%s' % uuidutils.generate_uuid()
 
 
+def is_load_import(content_type, url_path):
+    if (content_type == "multipart/form-data" and
+            url_path == "/v1/loads/import_load"):
+        return True
+    else:
+        return False
+
+
+class MultiFormDataHook(hooks.PecanHook):
+    """For multipart form-data, check disk space available before
+    proceeding.
+
+    Currently, it is only applying to import_load request, but
+    it can be extended to cover other multipart form-data requests
+    """
+
+    def on_route(self, state):
+        content_type = state.request.content_type
+        url_path = state.request.path
+        if is_load_import(content_type, url_path):
+            content_length = int(state.request.headers.get('Content-Length'))
+            # Currently, the restriction is 2x the file size:
+            # 1x from internal webob copy (see before override below)
+            # 1x from sysinv temporary copy
+            if not utils.is_space_available("/scratch", 2 * content_length):
+                msg = _(NO_SPACE_MSG + " on /scratch for request %s"
+                        % url_path)
+                raise webob.exc.HTTPInternalServerError(explanation=msg)
+
+    # Note: webob, for the multipart form-data request, creates 2 internal
+    # temporary copies, using the before override we can close the second
+    # temporary request before request goes to sysinv, this saves 1x file
+    # size required
+    def before(self, state):
+        content_type = state.request.content_type
+        url_path = state.request.path
+        if is_load_import(content_type, url_path):
+            state.request.body_file.close()
+
+
 class ConfigHook(hooks.PecanHook):
     """Attach the config object to the request so controllers can get to it."""
 
@@ -230,7 +271,13 @@ class AuditLogging(hooks.PecanHook):
 
         def json_post_data(rest_state):
             if 'form-data' in rest_state.request.headers.get('Content-Type'):
-                return " POST: {}".format(rest_state.request.params)
+                # rest_state.request.params causes an internal webob copy,
+                # prevent its call if there is no space available
+                size = int(rest_state.request.headers.get('Content-Length'))
+                if utils.is_space_available("/scratch", 2 * size):
+                    return " POST: {}".format(rest_state.request.params)
+                else:
+                    return " POST: " + NO_SPACE_MSG + " for processing"
             if not hasattr(rest_state.request, 'json'):
                 return ""
             return " POST: {}".format(rest_state.request.json)

sysinv/sysinv/sysinv/sysinv/api/middleware/parsable_error.py

@@ -31,6 +31,12 @@ from oslo_log import log
 
 LOG = log.getLogger(__name__)
 
+# As per webob.exc code:
+# https://github.com/Pylons/webob/blob/master/src/webob/exc.py
+# The explanation field is added to the HTTP exception as following:
+# ${explanation}<br /><br />
+WEBOB_EXPL_SEP = "<br /><br />"
+
 
 class ParsableErrorMiddleware(object):
     """Replace error body with something the client can parse.
@@ -86,7 +92,20 @@ class ParsableErrorMiddleware(object):
             else:
                 if six.PY3:
                     app_iter = [i.decode('utf-8') for i in app_iter]
-                body = [json.dumps({'error_message': '\n'.join(app_iter)})]
+                # Parse explanation field from webob.exc and add it as
+                # 'faultstring' to be processed by cgts-client
+                fault = None
+                app_data = '\n'.join(app_iter)
+                for data in app_data.split("\n"):
+                    if WEBOB_EXPL_SEP in str(data):
+                        # Remove separator, trailing and leading white spaces
+                        fault = str(data).replace(WEBOB_EXPL_SEP, "").strip()
+                        break
+                if fault is None:
+                    body = [json.dumps({'error_message': app_data})]
+                else:
+                    body = [json.dumps({'error_message':
+                                        json.dumps({'faultstring': fault})})]
                 if six.PY3:
                     body = [item.encode('utf-8') for item in body]
                 state['headers'].append(('Content-Type', 'application/json'))

sysinv/sysinv/sysinv/sysinv/common/utils.py

@@ -47,6 +47,7 @@ import keyring
 import math
 import os
 import pathlib
+import psutil
 import pwd
 import random
 import re
@@ -1279,6 +1280,14 @@ def is_cpe(host_obj):
             host_has_function(host_obj, constants.WORKER))
 
 
+def is_space_available(partition, size):
+    """
+        Returns if the given size is available in the specified partition
+    """
+    available_space = psutil.disk_usage(partition).free
+    return False if available_space < size else True
+
+
 def output_to_dict(output):
     dict = {}
     output = [_f for _f in output.split('\n') if _f]