From 978282410f754c4d66500cccf86d8718cdd0254a Mon Sep 17 00:00:00 2001 From: Zhong Luyao Date: Fri, 24 Mar 2017 16:42:40 +0800 Subject: [PATCH] Save instance_fault info when error Catch exception during process of deleting/creating/rebuild/ instances, and save fault info into db. Change-Id: Ia85c00521d754e0b1de0c650ebfb51fdbd5e06c9 --- mogan/common/utils.py | 102 ++++++++++++++++++++++++++++++++++++++++ mogan/engine/manager.py | 31 ++++++++++++ 2 files changed, 133 insertions(+) diff --git a/mogan/common/utils.py b/mogan/common/utils.py index 80595080..71bc5c6d 100644 --- a/mogan/common/utils.py +++ b/mogan/common/utils.py @@ -15,14 +15,20 @@ """Utilities and helper functions.""" +import functools +import inspect import re +import traceback from oslo_concurrency import lockutils from oslo_log import log as logging +from oslo_utils import encodeutils import six from mogan.common import exception from mogan.common import states +from mogan import objects + LOG = logging.getLogger(__name__) @@ -107,3 +113,99 @@ def process_event(fsm, instance, event=None): fsm.process_event(event) instance.status = fsm.current_state instance.save() + + +def get_wrapped_function(function): + """Get the method at the bottom of a stack of decorators.""" + if not hasattr(function, '__closure__') or not function.__closure__: + return function + + def _get_wrapped_function(function): + if not hasattr(function, '__closure__') or not function.__closure__: + return None + + for closure in function.__closure__: + func = closure.cell_contents + + deeper_func = _get_wrapped_function(func) + if deeper_func: + return deeper_func + elif hasattr(closure.cell_contents, '__call__'): + return closure.cell_contents + + return _get_wrapped_function(function) + + +def expects_func_args(*args): + def _decorator_checker(dec): + @functools.wraps(dec) + def _decorator(f): + base_f = get_wrapped_function(f) + arg_names, a, kw, _default = inspect.getargspec(base_f) + if a or kw or set(args) <= set(arg_names): + return dec(f) + else: + raise TypeError("Decorated function %(f_name)s does not " + "have the arguments expected by the " + "decorator %(d_name)s" % + {'f_name': base_f.__name__, + 'd_name': dec.__name__}) + return _decorator + return _decorator_checker + + +def _get_fault_detail(exc_info, error_code): + details = '' + if exc_info and error_code == 500: + tb = exc_info[2] + if tb: + details = ''.join(traceback.format_tb(tb)) + return six.text_type(details) + + +def safe_truncate(value, length): + """Safely truncates unicode strings such that their encoded length is + no greater than the length provided. + """ + b_value = encodeutils.safe_encode(value)[:length] + + decode_ok = False + # NOTE[ZhongLuyao] UTF-8 character byte size varies from 1 to 6. If + # truncating a long byte string to 255, the last character may be + # cut in the middle, so that UnicodeDecodeError will occur when + # converting it back to unicode. + while not decode_ok: + try: + u_value = encodeutils.safe_decode(b_value) + decode_ok = True + except UnicodeDecodeError: + b_value = b_value[:-1] + return u_value + + +def add_instance_fault_from_exc(context, instance, fault, exc_info=None, + fault_message=None): + """Adds the specified fault to the database.""" + code = 500 + if hasattr(fault, "kwargs"): + code = fault.kwargs.get('code', 500) + try: + if not fault_message: + fault_message = fault.format_message() + except Exception: + try: + fault_message = six.text_type(fault) + except Exception: + fault_message = None + if not fault_message: + fault_message = fault.__class__.__name__ + message = safe_truncate(fault_message, 255) + fault_dict = dict(exception=fault) + fault_dict["message"] = message + fault_dict["code"] = code + fault_obj = objects.InstanceFault(context=context) + fault_obj.instance_uuid = instance.uuid + fault_obj.update(fault_dict) + code = fault_obj.code + fault_obj.detail = _get_fault_detail(exc_info, code) + fault_obj.create() diff --git a/mogan/engine/manager.py b/mogan/engine/manager.py index cb9454d0..bf922d6d 100644 --- a/mogan/engine/manager.py +++ b/mogan/engine/manager.py @@ -13,6 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. +import functools +import sys + from oslo_log import log import oslo_messaging as messaging from oslo_service import periodic_task @@ -35,6 +38,31 @@ from mogan.objects import quota LOG = log.getLogger(__name__) +@utils.expects_func_args('instance') +def wrap_instance_fault(function): + """Wraps a method to catch exceptions related to instances. + + This decorator wraps a method to catch any exceptions having to do with + an instance that may get thrown. It then logs an instance fault in the db. + """ + + @functools.wraps(function) + def decorated_function(self, context, *args, **kwargs): + try: + return function(self, context, *args, **kwargs) + except exception.InstanceNotFound: + raise + except Exception as e: + kwargs.update(dict(zip(function.__code__.co_varnames[2:], args))) + + with excutils.save_and_reraise_exception(): + utils.add_instance_fault_from_exc(context, + kwargs['instance'], + e, sys.exc_info()) + + return decorated_function + + class EngineManager(base_manager.BaseEngineManager): """Mogan Engine manager main class.""" @@ -310,6 +338,7 @@ class EngineManager(base_manager.BaseEngineManager): if reservations: self.quota.commit(context, reservations) + @wrap_instance_fault def create_instance(self, context, instance, requested_networks, request_spec=None, filter_properties=None): """Perform a deployment.""" @@ -418,6 +447,7 @@ class EngineManager(base_manager.BaseEngineManager): self.driver.unplug_vifs(context, instance) self.driver.destroy(context, instance) + @wrap_instance_fault def delete_instance(self, context, instance): """Delete an instance.""" LOG.debug("Deleting instance...") @@ -477,6 +507,7 @@ class EngineManager(base_manager.BaseEngineManager): self.driver.rebuild(context, instance) + @wrap_instance_fault def rebuild_instance(self, context, instance): """Destroy and re-make this instance.