Fix race condition with router creation

Previously when murano deployed a bunch of environments from scratch
it was possible to have an error during router creation. Router was
created by env_1 and before it was fully configure it was accessed by
env_2 and then the error occurred.

Change-Id: I6dcd281a24825bb9c2ae09f45009faa6eca5a451
Closes-Bug: #1493883

murano/common/exceptions.py

@@ -62,3 +62,7 @@ class TimeoutException(Exception):
 
 class PolicyViolationException(Exception):
     pass
+
+
+class RouterInfoException(Exception):
+    pass

murano/engine/system/net_explorer.py

@@ -19,7 +19,9 @@ from netaddr.strategy import ipv4
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import uuidutils
+import retrying
 
+from murano.common import exceptions as exc
 from murano.common.i18n import _LI
 from murano.dsl import dsl
 from murano.dsl import helpers
@@ -37,6 +39,12 @@ class NetworkExplorer(object):
         self._settings = CONF.networking
         self._available_cidrs = self._generate_possible_cidrs()
 
+    # NOTE(starodubcevna): to avoid simultaneous router requests we use retry
+    # decorator with random delay 1-10 seconds between attempts and maximum
+    # delay time 30 seconds.
+    @retrying.retry(retry_on_exception=exc.RouterInfoException,
+                    wait_random_min=1000, wait_random_max=10000,
+                    stop_max_delay=30000)
     def get_default_router(self):
         client = self._clients.get_neutron_client()
         router_name = self._settings.router_name
@@ -75,8 +83,8 @@ class NetworkExplorer(object):
                 raise KeyError('Router %s was not found' % router_name)
         else:
             if routers[0]['external_gateway_info'] is None:
-                raise Exception('Please set external gateway '
-                                'for the router %s ' % router_name)
+                raise exc.RouterInfoException('Please set external gateway for'
+                                              ' the router %s ' % router_name)
             router_id = routers[0]['id']
         return router_id
 

requirements.txt

@@ -10,6 +10,7 @@ eventlet>=0.17.4
 PasteDeploy>=1.5.0
 Routes!=2.0,!=2.1,>=1.12.3;python_version=='2.7'
 Routes!=2.0,>=1.12.3;python_version!='2.7'
+retrying>=1.2.3,!=1.3.0 # Apache-2.0
 WebOb>=1.2.3
 kombu>=3.0.7
 psutil<2.0.0,>=1.1.1