Updating Thread lock with timeout

Change-Id: Iad54be361b6bbaee77561999efc2b776eee3784e

cloudpulse/TestManager/TestManager.py

@@ -132,28 +132,32 @@ class TestManager(object):
         Test = kwargs['test']
         func = self.command_ref[Test['name']]
         Test['state'] = 'running'
-        with cpulse_lock.thread_lock(Test, self.conductor_id):
-            LOG.info(('Updating db entry for the test %s') % Test['name'])
-            self.update_test(Test['uuid'], Test)
-        LOG.debug(('Running Test %s') % Test['name'])
-
         try:
-            result = func()
-        except Exception as e:
-            result = [404, str(e)]
+            with cpulse_lock.thread_lock(Test, self.conductor_id):
+                LOG.info(('Updating db entry for the test %s') % Test['name'])
+                self.update_test(Test['uuid'], Test)
+            LOG.debug(('Running Test %s') % Test['name'])
 
-        if result[0] == 200:
-            Test['state'] = 'success'
-            Test['result'] = textwrap.fill(str(result[1]), 40)
-        else:
-            Test['state'] = 'failed'
-            Test['result'] = textwrap.fill(str(result[1]), 40)
+            try:
+                result = func()
+            except Exception as e:
+                result = [404, str(e)]
+
+            if result[0] == 200:
+                Test['state'] = 'success'
+                Test['result'] = textwrap.fill(str(result[1]), 40)
+            else:
+                Test['state'] = 'failed'
+                Test['result'] = textwrap.fill(str(result[1]), 40)
+
+            LOG.debug(('Test State %s for test %s') %
+                      (Test['state'], Test['name']))
 
-        LOG.debug(('Test State %s for test %s') %
-                  (Test['state'], Test['name']))
-        with cpulse_lock.thread_lock(Test, self.conductor_id):
             LOG.info(('Updating db entry for the test %s') % Test['name'])
             self.update_test(Test['uuid'], Test)
+        except Exception as e:
+            LOG.debug("Not Running test in this run")
+            return
 
     def update_test(self, tuuid, patch):
         npatch = {}

cloudpulse/conductor/cpulse_lock.py

@@ -16,6 +16,7 @@ from cloudpulse import objects
 import contextlib
 import logging
 from oslo_utils import excutils
+import time
 
 LOG = logging.getLogger(__name__)
 
@@ -26,13 +27,15 @@ class CpulseLock(object):
         self.cpulse_test = cpulse_test
         self.conductor_id = conductor_id
 
-    def acquire(self, retry=True):
-        lock_conductor_id = objects.CpulseLock.create(self.cpulse_test.name,
-                                                      self.conductor_id)
-        if lock_conductor_id is None:
-            return
-        else:
-            raise exception.TestLocked(uuid=self.cpulse_test.name)
+    def acquire(self, retry=True, times=10):
+        for num in xrange(0, times):
+            lock_id = objects.CpulseLock.create(self.cpulse_test.name,
+                                                self.conductor_id)
+            if lock_id is None:
+                return
+            else:
+                time.sleep(3)
+        raise exception.TestLocked(uuid=self.cpulse_test.name)
 
     def release(self, test_name):
         result = objects.CpulseLock.release(test_name, self.conductor_id)