Merge "Add flexible wait mechanism for events"

freezer/exceptions/utils.py

@@ -0,0 +1,7 @@
+class TimeoutException(Exception):
+    msg = "Timeout has been occured."
+
+    def __init__(self, message=None, **kwargs):
+        if not message:
+            message = self.msg
+        super(TimeoutException, self).__init__(message, kwargs)

freezer/openstack/backup.py

@@ -15,7 +15,6 @@ limitations under the License.
 
 Freezer Backup modes related functions
 """
-import time
 
 from oslo_log import log
 
@@ -51,20 +50,36 @@ class BackupOs(object):
         instance = nova.servers.get(instance_id)
         glance = client_manager.get_glance()
 
-        if instance.__dict__['OS-EXT-STS:task_state']:
-            time.sleep(5)
-            instance = nova.servers.get(instance)
+        def instance_finish_task():
+            instance = nova.servers.get(instance_id)
+            return not instance.__dict__['OS-EXT-STS:task_state']
+
+        utils.wait_for(
+            instance_finish_task, 1, 10,
+            message="Waiting for instance {0} to finish {1} to start the "
+                    "snapshot process".format(
+                        instance_id,
+                        instance.__dict__['OS-EXT-STS:task_state']
+                    )
+        )
+        instance = nova.servers.get(instance)
 
         image_id = nova.servers.create_image(instance,
                                              "snapshot_of_%s" % instance_id)
 
         image = glance.images.get(image_id)
-        while image.status != 'active':
-            time.sleep(5)
-            try:
-                image = glance.images.get(image_id)
-            except Exception as e:
-                LOG.error(e)
+
+        def image_active():
+            image = glance.images.get(image_id)
+            return image.status == 'active'
+
+        utils.wait_for(image_active, 1, 10,
+                       message="Waiting for instnace {0} snapshot to become "
+                               "active".format(instance_id))
+        try:
+            image = glance.images.get(image_id)
+        except Exception as e:
+            LOG.error(e)
 
         stream = client_manager.download_image(image)
         package = "{0}/{1}".format(instance_id, utils.DateTime.now().timestamp)

freezer/tests/unit/utils/test_utils.py

@@ -17,8 +17,11 @@ import datetime
 import os
 import time
 
+import mock
+
 from mock import patch
 
+from freezer.exceptions import utils as exception_utils
 from freezer.openstack import osclients
 from freezer.tests import commons
 from freezer.utils import utils
@@ -202,3 +205,13 @@ class TestDateTime(object):
         d2 = datetime.datetime(2015, 3, 7, 18, 18, 38, 508411)
         ts2 = utils.DateTime(d2)
         assert '0:30:53.791612' == '{}'.format(ts2 - self.datetime)
+
+    def test_wait_for_positive(self):
+        condition = mock.MagicMock(side_effect=[False, False, True, True])
+        utils.wait_for(condition, 0.1, 1)
+        self.assertEqual(3, condition.called)
+
+    def test_wait_for_negative(self):
+        condition = mock.MagicMock(side_effect=[False, False, False])
+        self.assertRaises(exception_utils.TimeoutException,
+                          utils.wait_for(condition, 0.1, 0.2))

freezer/utils/utils.py

@@ -25,11 +25,11 @@ import sys
 import time
 
 from distutils import spawn as distspawn
+from freezer.exceptions import utils
 from functools import wraps
 from oslo_log import log
 from six.moves import configparser
 
-
 LOG = log.getLogger(__name__)
 
 
@@ -512,3 +512,14 @@ def set_max_process_priority():
         ])
     except Exception as priority_error:
         LOG.warning('Priority: {0}'.format(priority_error))
+
+
+def wait_for(condition_func, wait_interval, timeout, message=None):
+    while timeout > 0:
+        if condition_func():
+            return
+
+        time.sleep(wait_interval)
+        timeout -= wait_interval
+
+    raise utils.TimeoutException(message)