Merge "Reschedule queries to nova-scheduler after a timeout occurs"

nova/scheduler/client/__init__.py

@@ -17,6 +17,8 @@ import functools
 
 from oslo.utils import importutils
 
+from nova.scheduler import utils
+
 
 class LazyLoader(object):
 
@@ -44,6 +46,7 @@ class SchedulerClient(object):
         self.reportclient = LazyLoader(importutils.import_class(
             'nova.scheduler.client.report.SchedulerReportClient'))
 
+    @utils.retry_select_destinations
     def select_destinations(self, context, request_spec, filter_properties):
         return self.queryclient.select_destinations(
             context, request_spec, filter_properties)

nova/scheduler/utils.py

@@ -15,9 +15,11 @@
 """Utility methods for scheduling."""
 
 import collections
+import functools
 import sys
 
 from oslo.config import cfg
+from oslo import messaging
 from oslo.serialization import jsonutils
 
 from nova.compute import flavors
@@ -313,3 +315,34 @@ def setup_instance_group(context, request_spec, filter_properties):
         filter_properties['group_updated'] = True
         filter_properties['group_hosts'] = group_info.hosts
         filter_properties['group_policies'] = group_info.policies
+
+
+def retry_on_timeout(retries=1):
+    """Retry the call in case a MessagingTimeout is raised.
+
+    A decorator for retrying calls when a service dies mid-request.
+
+    :param retries: Number of retries
+    :returns: Decorator
+    """
+    def outer(func):
+        @functools.wraps(func)
+        def wrapped(*args, **kwargs):
+            attempt = 0
+            while True:
+                try:
+                    return func(*args, **kwargs)
+                except messaging.MessagingTimeout:
+                    attempt += 1
+                    if attempt <= retries:
+                        LOG.warning(_LW(
+                            "Retrying %(name)s after a MessagingTimeout, "
+                            "attempt %(attempt)s of %(retries)s."),
+                                 {'attempt': attempt, 'retries': retries,
+                                  'name': func.__name__})
+                    else:
+                        raise
+        return wrapped
+    return outer
+
+retry_select_destinations = retry_on_timeout(_max_attempts() - 1)

nova/tests/unit/scheduler/test_client.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 import mock
+from oslo import messaging
 
 from nova.conductor import api as conductor_api
 from nova import context
@@ -101,6 +102,25 @@ class SchedulerClientTestCase(test.TestCase):
         mock_select_destinations.assert_called_once_with(
             'ctxt', 'fake_spec', 'fake_prop')
 
+    @mock.patch.object(scheduler_query_client.SchedulerQueryClient,
+                       'select_destinations',
+                       side_effect=messaging.MessagingTimeout())
+    def test_select_destinations_timeout(self, mock_select_destinations):
+        # check if the scheduler service times out properly
+        fake_args = ['ctxt', 'fake_spec', 'fake_prop']
+        self.assertRaises(messaging.MessagingTimeout,
+                          self.client.select_destinations, *fake_args)
+        mock_select_destinations.assert_has_calls([mock.call(*fake_args)] * 2)
+
+    @mock.patch.object(scheduler_query_client.SchedulerQueryClient,
+                       'select_destinations', side_effect=[
+                           messaging.MessagingTimeout(), mock.DEFAULT])
+    def test_select_destinations_timeout_once(self, mock_select_destinations):
+        # scenario: the scheduler service times out & recovers after failure
+        fake_args = ['ctxt', 'fake_spec', 'fake_prop']
+        self.client.select_destinations(*fake_args)
+        mock_select_destinations.assert_has_calls([mock.call(*fake_args)] * 2)
+
     @mock.patch.object(scheduler_report_client.SchedulerReportClient,
                        'update_resource_stats')
     def test_update_resource_stats(self, mock_update_resource_stats):