Corrected max secs for concurrent trans retries

This was most likely meant as a max 2s delay here, not a max 2ms
delay.

Also includes a related change: when retries for metadata updates are
attempted, make sure we do not have a stale value of the atomic_key
(otherwise we'll just inevitably hit the ConcurrentTransaction issue).

Co-Authored-By: Crag Wolfe <cwolfe@redhat.com>
Partial-Bug: #1651768
Change-Id: Ie56e0e4ff93633db1f4752859d2b2a9506922911

heat/engine/service_software_config.py

@@ -100,7 +100,10 @@ class SoftwareConfigService(object):
     def _push_metadata_software_deployments(
             self, cnxt, server_id, stack_user_project_id):
         rs = db_api.resource_get_by_physical_resource_id(cnxt, server_id)
-        if not rs or rs.action == resource.Resource.DELETE:
+        if not rs:
+            return
+        cnxt.session.refresh(rs)
+        if rs.action == resource.Resource.DELETE:
             return
         deployments = self.metadata_software_deployments(cnxt, server_id)
         md = rs.rsrc_metadata or {}

heat/objects/resource.py

@@ -41,7 +41,7 @@ LOG = logging.getLogger(__name__)
 def retry_on_conflict(func):
     wrapper = tenacity.retry(
         stop=tenacity.stop_after_attempt(11),
-        wait=tenacity.wait_random(max=0.002),
+        wait=tenacity.wait_random(max=2),
         retry=tenacity.retry_if_exception_type(
             exception.ConcurrentTransaction),
         reraise=True)

heat/tests/engine/service/test_software_config.py

@@ -649,8 +649,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'deployments': {'deploy': 'this'}
         }
 
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', None)
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
         put.side_effect = Exception('Unexpected requests.put')
@@ -674,12 +675,14 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
         deployments = {'deploy': 'this'}
         md_sd.return_value = deployments
 
-        self.assertRaises(
-            exception.ConcurrentTransaction,
-            self.engine.software_config._push_metadata_software_deployments,
-            self.ctx,
-            '1234',
-            None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            f = self.engine.software_config._push_metadata_software_deployments
+            self.assertRaises(
+                exception.ConcurrentTransaction,
+                f,
+                self.ctx,
+                '1234',
+                None)
         # retry ten times then the final failure
         self.assertEqual(11, res_upd.call_count)
         put.assert_not_called()
@@ -709,9 +712,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'original': 'metadata',
             'deployments': {'deploy': 'this'}
         }
-
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', None)
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
 
@@ -748,8 +751,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'deployments': {'deploy': 'this'}
         }
 
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', 'project1')
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', 'project1')
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
 
@@ -917,8 +921,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
         zaqar_client.queue.return_value = queue
         queue.pop.return_value = [mock.Mock(body='ok')]
 
-        deployment = self._create_software_deployment(
-            status='IN_PROGRESS', config_id=config['id'])
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            deployment = self._create_software_deployment(
+                status='IN_PROGRESS', config_id=config['id'])
 
         deployment_id = deployment['id']
         self.assertEqual(