Merge "Omit RescheduledException in instance_fault.message"

nova/compute/manager.py

@@ -1913,13 +1913,17 @@ class ComputeManager(manager.Manager):
                 self._cleanup_allocated_networks(context, instance,
                     requested_networks)
                 compute_utils.add_instance_fault_from_exc(context,
-                        instance, e, sys.exc_info())
+                        instance, e, sys.exc_info(),
+                        fault_message=e.kwargs['reason'])
                 self._nil_out_instance_obj_host_and_node(instance)
                 self._set_instance_obj_error_state(context, instance,
                                                    clean_task_state=True)
                 return build_results.FAILED
             LOG.debug(e.format_message(), instance=instance)
+            # This will be used for logging the exception
             retry['exc'] = traceback.format_exception(*sys.exc_info())
+            # This will be used for setting the instance fault message
+            retry['exc_reason'] = e.kwargs['reason']
             # NOTE(comstud): Deallocate networks if the driver wants
             # us to do so.
             if self.driver.deallocate_networks_on_reschedule(instance):

nova/compute/utils.py

@@ -41,7 +41,7 @@ CONF.import_opt('host', 'nova.netconf')
 LOG = log.getLogger(__name__)
 
 
-def exception_to_dict(fault):
+def exception_to_dict(fault, message=None):
     """Converts exceptions to a dict for use in notifications."""
     # TODO(johngarbutt) move to nova/exception.py to share with wrap_exception
 
@@ -52,7 +52,8 @@ def exception_to_dict(fault):
     # get the message from the exception that was thrown
     # if that does not exist, use the name of the exception class itself
     try:
-        message = fault.format_message()
+        if not message:
+            message = fault.format_message()
     # These exception handlers are broad so we don't fail to log the fault
     # just because there is an unexpected error retrieving the message
     except Exception:
@@ -81,13 +82,14 @@ def _get_fault_details(exc_info, error_code):
     return six.text_type(details)
 
 
-def add_instance_fault_from_exc(context, instance, fault, exc_info=None):
+def add_instance_fault_from_exc(context, instance, fault, exc_info=None,
+                                fault_message=None):
     """Adds the specified fault to the database."""
 
     fault_obj = objects.InstanceFault(context=context)
     fault_obj.host = CONF.host
     fault_obj.instance_uuid = instance.uuid
-    fault_obj.update(exception_to_dict(fault))
+    fault_obj.update(exception_to_dict(fault, message=fault_message))
     code = fault_obj.code
     fault_obj.details = _get_fault_details(exc_info, code)
     fault_obj.create()

nova/scheduler/utils.py

@@ -166,15 +166,15 @@ def populate_retry(filter_properties, instance_uuid):
     retry['num_attempts'] += 1
 
     _log_compute_error(instance_uuid, retry)
-    exc = retry.pop('exc', None)
+    exc_reason = retry.pop('exc_reason', None)
 
     if retry['num_attempts'] > max_attempts:
         msg = (_('Exceeded max scheduling attempts %(max_attempts)d '
                  'for instance %(instance_uuid)s. '
-                 'Last exception: %(exc)s')
+                 'Last exception: %(exc_reason)s')
                % {'max_attempts': max_attempts,
                   'instance_uuid': instance_uuid,
-                  'exc': exc})
+                  'exc_reason': exc_reason})
         raise exception.MaxRetriesExceeded(reason=msg)
 
 

nova/tests/unit/compute/test_compute.py

@@ -6165,6 +6165,37 @@ class ComputeTestCase(BaseTestCase):
                                                   instance,
                                                   NotImplementedError(message))
 
+    def test_add_instance_fault_with_message(self):
+        instance = self._create_fake_instance_obj()
+        exc_info = None
+
+        def fake_db_fault_create(ctxt, values):
+            self.assertIn('raise NotImplementedError', values['details'])
+            del values['details']
+
+            expected = {
+                'code': 500,
+                'message': 'hoge',
+                'instance_uuid': instance['uuid'],
+                'host': self.compute.host
+            }
+            self.assertEqual(expected, values)
+            return self._fill_fault(expected)
+
+        try:
+            raise NotImplementedError('test')
+        except NotImplementedError:
+            exc_info = sys.exc_info()
+
+        self.stubs.Set(nova.db, 'instance_fault_create', fake_db_fault_create)
+
+        ctxt = context.get_admin_context()
+        compute_utils.add_instance_fault_from_exc(ctxt,
+                                                  instance,
+                                                  NotImplementedError('test'),
+                                                  exc_info,
+                                                  fault_message='hoge')
+
     def _test_cleanup_running(self, action):
         admin_context = context.get_admin_context()
         deleted_at = (timeutils.utcnow() -
@@ -7386,6 +7417,26 @@ class ComputeTestCase(BaseTestCase):
         mock_snapshot_get.assert_any_call(mock.ANY, 'fake-id2')
         self.assertEqual(4, mock_snapshot_get.call_count)
 
+    def test_instance_fault_message_no_rescheduled_details_without_retry(self):
+        """This test simulates a spawn failure with no retry data.
+
+        If driver spawn raises an exception and there is no retry data
+        available, the instance fault message should not contain any details
+        about rescheduling. The fault message field is limited in size and a
+        long message about rescheduling displaces the original error message.
+        """
+        class TestException(Exception):
+            pass
+
+        instance = self._create_fake_instance_obj()
+
+        with mock.patch.object(self.compute.driver, 'spawn') as mock_spawn:
+            mock_spawn.side_effect = TestException('Preserve this')
+            self.compute.build_and_run_instance(
+                    self.context, instance, {}, {}, {},
+                    block_device_mapping=[])
+        self.assertEqual('Preserve this', instance.fault.message)
+
 
 class ComputeAPITestCase(BaseTestCase):
     def setUp(self):

nova/tests/unit/compute/test_compute_mgr.py

@@ -3199,7 +3199,8 @@ class ComputeManagerBuildInstanceTestCase(test.NoDBTestCase):
         self.compute._cleanup_allocated_networks(self.context, self.instance,
             self.requested_networks)
         compute_utils.add_instance_fault_from_exc(self.context, self.instance,
-                mox.IgnoreArg(), mox.IgnoreArg())
+                mox.IgnoreArg(), mox.IgnoreArg(),
+                fault_message=mox.IgnoreArg())
         self.compute._nil_out_instance_obj_host_and_node(self.instance)
         self.compute._set_instance_obj_error_state(self.context, self.instance,
                                                    clean_task_state=True)

nova/tests/unit/scheduler/test_scheduler_utils.py

@@ -192,7 +192,7 @@ class SchedulerUtilsTestCase(test.NoDBTestCase):
         _max_attempts.return_value = 2
         msg = 'The exception text was preserved!'
         filter_properties = dict(retry=dict(num_attempts=2, hosts=[],
-                                            exc=[msg]))
+                                            exc_reason=[msg]))
         nvh = self.assertRaises(exception.MaxRetriesExceeded,
                                 scheduler_utils.populate_retry,
                                 filter_properties, 'fake-uuid')