Retry on connection error to neutron

In case of keystoneauth1 ConnectError, manila will retry the neutron
API call. For create_port(), it will make sure no duplicate ports
are created.

Closes-bug: #2049507
Change-Id: I12fece58671e9fb3705e22090187c42d9c3a74d9

manila/network/neutron/api.py

@@ -14,6 +14,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from keystoneauth1 import exceptions as ks_exec
 from keystoneauth1 import loading as ks_loading
 from neutronclient.common import exceptions as neutron_client_exc
 from neutronclient.v2_0 import client as clientv20
@@ -24,6 +25,8 @@ from manila.common import client_auth
 from manila import context
 from manila import exception
 from manila.network.neutron import constants as neutron_constants
+from manila import utils
+
 
 NEUTRON_GROUP = 'neutron'
 
@@ -122,6 +125,20 @@ class API(object):
                     fixed_ip=None, device_owner=None, device_id=None,
                     mac_address=None, port_security_enabled=True,
                     security_group_ids=None, dhcp_opts=None, **kwargs):
+        return self._create_port(tenant_id, network_id, host_id=host_id,
+                                 subnet_id=subnet_id, fixed_ip=fixed_ip,
+                                 device_owner=device_owner,
+                                 device_id=device_id, mac_address=mac_address,
+                                 port_security_enabled=port_security_enabled,
+                                 security_group_ids=security_group_ids,
+                                 dhcp_opts=dhcp_opts, **kwargs)
+
+    @utils.retry(retry_param=ks_exec.ConnectFailure, retries=5)
+    def _create_port(self, tenant_id, network_id, host_id=None, subnet_id=None,
+                     fixed_ip=None, device_owner=None, device_id=None,
+                     mac_address=None, port_security_enabled=True,
+                     security_group_ids=None, dhcp_opts=None, name=None,
+                     **kwargs):
         try:
             port_req_body = {'port': {}}
             port_req_body['port']['network_id'] = network_id
@@ -152,6 +169,8 @@ class API(object):
                 port_req_body['port']['device_owner'] = device_owner
             if device_id:
                 port_req_body['port']['device_id'] = device_id
+            if name:
+                port_req_body['port']['name'] = name
             if kwargs:
                 port_req_body['port'].update(kwargs)
             port = self.client.create_port(port_req_body).get('port', {})
@@ -163,6 +182,19 @@ class API(object):
                 raise exception.PortLimitExceeded()
             raise exception.NetworkException(code=e.status_code,
                                              message=e.message)
+        except ks_exec.ConnectFailure:
+            LOG.warning('Create Port: Neutron connection failure')
+            # check if port is created in neutron else re-raise connectFailure
+            search_opts = {
+                'device_id': device_id,
+                'network_id': network_id,
+                'name': name
+            }
+            try:
+                ports = self.list_ports(**search_opts)
+                return ports[0]
+            except ks_exec.ConnectFailure as kse:
+                raise kse
 
     def delete_port(self, port_id):
         try:
@@ -182,6 +214,7 @@ class API(object):
         """List ports for the client based on search options."""
         return self.client.list_ports(**search_opts).get('ports')
 
+    @utils.retry(retry_param=ks_exec.ConnectFailure, retries=5)
     def show_port(self, port_id):
         """Return the port for the client given the port id."""
         try:
@@ -189,11 +222,14 @@ class API(object):
         except neutron_client_exc.NeutronClientException as e:
             raise exception.NetworkException(code=e.status_code,
                                              message=e.message)
+        except ks_exec.ConnectFailure as e:
+            raise e
 
     def get_all_networks(self):
         """Get all networks for client."""
         return self.client.list_networks().get('networks')
 
+    @utils.retry(retry_param=ks_exec.ConnectFailure, retries=5)
     def get_network(self, network_uuid):
         """Get specific network for client."""
         try:
@@ -202,6 +238,8 @@ class API(object):
         except neutron_client_exc.NeutronClientException as e:
             raise exception.NetworkException(code=e.status_code,
                                              message=e.message)
+        except ks_exec.ConnectFailure as e:
+            raise e
 
     def get_subnet(self, subnet_uuid):
         """Get specific subnet for client."""

manila/network/neutron/neutron_network_plugin.py

@@ -162,10 +162,11 @@ class NeutronNetworkPlugin(network.NetworkBaseAPI):
         device_owner = kwargs.get('device_owner', 'share')
 
         ports = []
-        for __ in range(0, allocation_count):
+        for current_count in range(0, allocation_count):
             ports.append(self._create_port(
                          context, share_server, share_network,
-                         share_network_subnet, device_owner))
+                         share_network_subnet, device_owner,
+                         current_count))
 
         return ports
 
@@ -312,18 +313,19 @@ class NeutronNetworkPlugin(network.NetworkBaseAPI):
             self._delete_port(context, port)
 
     def _get_port_create_args(self, share_server, share_network_subnet,
-                              device_owner):
+                              device_owner, count=0):
         return {
             "network_id": share_network_subnet['neutron_net_id'],
             "subnet_id": share_network_subnet['neutron_subnet_id'],
             "device_owner": 'manila:' + device_owner,
             "device_id": share_server.get('id'),
+            "name": share_server.get('id') + '_' + str(count)
         }
 
     def _create_port(self, context, share_server, share_network,
-                     share_network_subnet, device_owner):
+                     share_network_subnet, device_owner, count=0):
         create_args = self._get_port_create_args(
-            share_server, share_network_subnet, device_owner)
+            share_server, share_network_subnet, device_owner, count)
 
         port = self.neutron_api.create_port(
             share_network['project_id'], **create_args)
@@ -577,10 +579,10 @@ class NeutronBindNetworkPlugin(NeutronNetworkPlugin):
         raise exception.NetworkBindException(msg)
 
     def _get_port_create_args(self, share_server, share_network_subnet,
-                              device_owner):
+                              device_owner, count=0):
         arguments = super(
             NeutronBindNetworkPlugin, self)._get_port_create_args(
-            share_server, share_network_subnet, device_owner)
+            share_server, share_network_subnet, device_owner, count)
         arguments['host_id'] = self.config.neutron_host_id
         arguments['binding:vnic_type'] = self.config.neutron_vnic_type
         if self.binding_profiles:

manila/tests/network/neutron/test_neutron_plugin.py

@@ -252,7 +252,8 @@ class NeutronNetworkPluginTest(test.TestCase):
                 network_id=fake_share_network_subnet['neutron_net_id'],
                 subnet_id=fake_share_network_subnet['neutron_subnet_id'],
                 device_owner='manila:share',
-                device_id=fake_share_network['id'])
+                device_id=fake_share_network['id'],
+                name=fake_share_network['id'] + '_0')
             db_api.network_allocation_create.assert_called_once_with(
                 self.fake_context,
                 fake_network_allocation)
@@ -289,13 +290,15 @@ class NeutronNetworkPluginTest(test.TestCase):
                     network_id=fake_share_network_subnet['neutron_net_id'],
                     subnet_id=fake_share_network_subnet['neutron_subnet_id'],
                     device_owner='manila:share',
-                    device_id=fake_share_network['id']),
+                    device_id=fake_share_network['id'],
+                    name=fake_share_network['id'] + '_0'),
                 mock.call(
                     fake_share_network['project_id'],
                     network_id=fake_share_network_subnet['neutron_net_id'],
                     subnet_id=fake_share_network_subnet['neutron_subnet_id'],
                     device_owner='manila:share',
-                    device_id=fake_share_network['id']),
+                    device_id=fake_share_network['id'],
+                    name=fake_share_network['id'] + '_1'),
             ]
             db_api_calls = [
                 mock.call(self.fake_context, fake_network_allocation),
@@ -978,6 +981,7 @@ class NeutronBindNetworkPluginTest(test.TestCase):
                 'subnet_id': fake_share_network_subnet['neutron_subnet_id'],
                 'device_owner': 'manila:share',
                 'device_id': fake_share_network['id'],
+                'name': fake_share_network['id'] + '_0',
             }
             self.bind_plugin.neutron_api.create_port.assert_called_once_with(
                 fake_share_network['project_id'], **expected_kwargs)
@@ -1059,7 +1063,8 @@ class NeutronBindNetworkPluginTest(test.TestCase):
                 'network_id': fake_share_network_multi['neutron_net_id'],
                 'subnet_id': fake_share_network_multi['neutron_subnet_id'],
                 'device_owner': 'manila:share',
-                'device_id': fake_share_network_multi['id']
+                'device_id': fake_share_network_multi['id'],
+                'name': fake_share_network['id'] + '_0',
             }
             self.bind_plugin.neutron_api.create_port.assert_called_once_with(
                 fake_share_network_multi['project_id'], **expected_kwargs)
@@ -1174,7 +1179,8 @@ class NeutronBindNetworkPluginTest(test.TestCase):
             'network_id': fake_share_network_subnet['neutron_net_id'],
             'subnet_id': fake_share_network_subnet['neutron_subnet_id'],
             'device_owner': 'manila:' + fake_device_owner,
-            'device_id': fake_share_server['id']
+            'device_id': fake_share_server['id'],
+            'name': fake_share_server['id'] + '_0',
         }
         if neutron_binding_profiles:
             expected_create_args['binding:profile'] = {
@@ -1228,7 +1234,8 @@ class NeutronBindNetworkPluginTest(test.TestCase):
             'network_id': fake_share_network_subnet['neutron_net_id'],
             'subnet_id': fake_share_network_subnet['neutron_subnet_id'],
             'device_owner': 'manila:' + fake_device_owner,
-            'device_id': fake_share_server['id']
+            'device_id': fake_share_server['id'],
+            'name': fake_share_server['id'] + '_0'
         }
 
         self.assertEqual(expected_create_args, create_args)
@@ -1505,6 +1512,7 @@ class NeutronBindSingleNetworkPluginTest(test.TestCase):
                 'subnet_id': fake_share_network_subnet['neutron_subnet_id'],
                 'device_owner': 'manila:share',
                 'device_id': fake_share_network['id'],
+                'name': fake_share_network['id'] + '_0',
             }
             self.bind_plugin.neutron_api.create_port.assert_called_once_with(
                 fake_share_network['project_id'], **expected_kwargs)
@@ -1611,7 +1619,8 @@ class NeutronBindSingleNetworkPluginTest(test.TestCase):
             'network_id': fake_share_network_subnet['neutron_net_id'],
             'subnet_id': fake_share_network_subnet['neutron_subnet_id'],
             'device_owner': 'manila:' + fake_device_owner,
-            'device_id': fake_share_server['id']
+            'device_id': fake_share_server['id'],
+            'name': fake_share_server['id'] + '_0',
         }
         if neutron_binding_profiles:
             expected_create_args['binding:profile'] = {
@@ -1665,7 +1674,8 @@ class NeutronBindSingleNetworkPluginTest(test.TestCase):
             'network_id': fake_share_network_subnet['neutron_net_id'],
             'subnet_id': fake_share_network_subnet['neutron_subnet_id'],
             'device_owner': 'manila:' + fake_device_owner,
-            'device_id': fake_share_server['id']
+            'device_id': fake_share_server['id'],
+            'name': fake_share_server['id'] + '_0'
         }
 
         self.assertEqual(expected_create_args, create_args)
@@ -1734,6 +1744,7 @@ class NeutronBindNetworkPluginWithNormalTypeTest(test.TestCase):
                 'subnet_id': fake_share_network_subnet['neutron_subnet_id'],
                 'device_owner': 'manila:share',
                 'device_id': fake_share_server['id'],
+                'name': fake_share_server['id'] + '_0',
             }
             self.bind_plugin.neutron_api.create_port.assert_called_once_with(
                 fake_share_network['project_id'], **expected_kwargs)
@@ -1823,6 +1834,7 @@ class NeutronBindSingleNetworkPluginWithNormalTypeTest(test.TestCase):
                 'subnet_id': fake_share_network_subnet['neutron_subnet_id'],
                 'device_owner': 'manila:share',
                 'device_id': fake_share_network['id'],
+                'name': fake_share_network['id'] + '_0',
             }
             self.bind_plugin.neutron_api.create_port.assert_called_once_with(
                 fake_share_network['project_id'], **expected_kwargs)

releasenotes/notes/bug-2049507-retry-on-connection-error-to-neutron-df7d2ddac5f30773.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Manila will retry neutron API calls e.g. create_port(), show_port() in
+    case of keystoneauth1 connection error. For more details, please refer to
+    `launchpad bug #2049507 <https://bugs.launchpad.net/manila/+bug/2049507>`_