From 416bad4325ac8311e7b9b891fb6d70c67a97ab95 Mon Sep 17 00:00:00 2001
From: Zane Bitter <zbitter@redhat.com>
Date: Thu, 26 May 2016 13:40:53 -0400
Subject: [PATCH] Corrected max secs for concurrent trans retries

This was most likely meant as a max 2s delay here, not a max 2ms
delay.

Also includes a related change: when retries for metadata updates are
attempted, make sure we do not have a stale value of the atomic_key
(otherwise we'll just inevitably hit the ConcurrentTransaction issue).

 Conflicts:
	heat/engine/service_software_config.py

Co-Authored-By: Crag Wolfe <cwolfe@redhat.com>
Partial-Bug: #1651768
Change-Id: Ie56e0e4ff93633db1f4752859d2b2a9506922911
(cherry picked from commit e37d9fab8fe2e779ae8c0e2311de2601b66c66b6)
---
 heat/engine/service_software_config.py        |  1 +
 heat/objects/resource.py                      |  2 +-
 .../engine/service/test_software_config.py    | 35 +++++++++++--------
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/heat/engine/service_software_config.py b/heat/engine/service_software_config.py
index 6c11f095c..5ea48fb6c 100644
--- a/heat/engine/service_software_config.py
+++ b/heat/engine/service_software_config.py
@@ -102,6 +102,7 @@ class SoftwareConfigService(object):
         rs = db_api.resource_get_by_physical_resource_id(cnxt, server_id)
         if not rs:
             return
+        cnxt.session.refresh(rs)
         deployments = self.metadata_software_deployments(cnxt, server_id)
         md = rs.rsrc_metadata or {}
         md['deployments'] = deployments
diff --git a/heat/objects/resource.py b/heat/objects/resource.py
index c1a365c2f..528400df2 100644
--- a/heat/objects/resource.py
+++ b/heat/objects/resource.py
@@ -42,7 +42,7 @@ LOG = logging.getLogger(__name__)
 def retry_on_conflict(func):
     wrapper = tenacity.retry(
         stop=tenacity.stop_after_attempt(11),
-        wait=tenacity.wait_random(max=0.002),
+        wait=tenacity.wait_random(max=2),
         retry=tenacity.retry_if_exception_type(
             exception.ConcurrentTransaction),
         reraise=True)
diff --git a/heat/tests/engine/service/test_software_config.py b/heat/tests/engine/service/test_software_config.py
index 32cb46ecb..a1e2c01f6 100644
--- a/heat/tests/engine/service/test_software_config.py
+++ b/heat/tests/engine/service/test_software_config.py
@@ -649,8 +649,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'deployments': {'deploy': 'this'}
         }
 
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', None)
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
         put.side_effect = Exception('Unexpected requests.put')
@@ -674,12 +675,14 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
         deployments = {'deploy': 'this'}
         md_sd.return_value = deployments
 
-        self.assertRaises(
-            exception.ConcurrentTransaction,
-            self.engine.software_config._push_metadata_software_deployments,
-            self.ctx,
-            '1234',
-            None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            f = self.engine.software_config._push_metadata_software_deployments
+            self.assertRaises(
+                exception.ConcurrentTransaction,
+                f,
+                self.ctx,
+                '1234',
+                None)
         # retry ten times then the final failure
         self.assertEqual(11, res_upd.call_count)
         put.assert_not_called()
@@ -709,9 +712,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'original': 'metadata',
             'deployments': {'deploy': 'this'}
         }
-
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', None)
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
 
@@ -748,8 +751,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'deployments': {'deploy': 'this'}
         }
 
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', 'project1')
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', 'project1')
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
 
@@ -917,8 +921,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
         zaqar_client.queue.return_value = queue
         queue.pop.return_value = [mock.Mock(body='ok')]
 
-        deployment = self._create_software_deployment(
-            status='IN_PROGRESS', config_id=config['id'])
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            deployment = self._create_software_deployment(
+                status='IN_PROGRESS', config_id=config['id'])
 
         deployment_id = deployment['id']
         self.assertEqual(