diff --git a/gbpservice/neutron/services/servicechain/drivers/simplechain_driver.py b/gbpservice/neutron/services/servicechain/drivers/simplechain_driver.py index 92132ae6e..716a0022b 100644 --- a/gbpservice/neutron/services/servicechain/drivers/simplechain_driver.py +++ b/gbpservice/neutron/services/servicechain/drivers/simplechain_driver.py @@ -11,6 +11,8 @@ # under the License. import ast +import time + from heatclient import client as heat_client from neutron.common import log from neutron.db import model_base @@ -27,8 +29,23 @@ from gbpservice.neutron.services.servicechain.common import exceptions as exc LOG = logging.getLogger(__name__) +service_chain_opts = [ + cfg.IntOpt('stack_delete_retries', + default=5, + help=_("Number of attempts to retry for stack deletion")), + cfg.IntOpt('stack_delete_retry_wait', + default=3, + help=_("Wait time between two successive stack delete " + "retries")), +] + + +cfg.CONF.register_opts(service_chain_opts, "servicechain") + # Service chain API supported Values sc_supported_type = [pconst.LOADBALANCER, pconst.FIREWALL] +STACK_DELETE_RETRIES = cfg.CONF.servicechain.stack_delete_retries +STACK_DELETE_RETRY_WAIT = cfg.CONF.servicechain.stack_delete_retry_wait class ServiceChainInstanceStack(model_base.BASEV2): @@ -248,8 +265,44 @@ class SimpleChainDriver(object): heatclient = HeatClient(context) for stack in stack_ids: heatclient.delete(stack.stack_id) + for stack in stack_ids: + self._wait_for_stack_delete(heatclient, stack.stack_id) self._delete_chain_stacks_db(context.session, instance_id) + # Wait for the heat stack to be deleted for a maximum of 15 seconds + # we check the status every 3 seconds and call sleep again + # This is required because cleanup of subnet fails when the stack created + # some ports on the subnet and the resource delete is not completed by + # the time subnet delete is triggered by Resource Mapping driver + def _wait_for_stack_delete(self, heatclient, stack_id): + stack_delete_retries = STACK_DELETE_RETRIES + while True: + try: + stack = heatclient.get(stack_id) + if stack.stack_status == 'DELETE_COMPLETE': + return + elif stack.stack_status == 'ERROR': + heatclient.delete(stack_id) + except Exception: + LOG.exception(_("Service Chain Instance cleanup may not have " + "happened because Heat API request failed " + "while waiting for the stack %(stack)s to be " + "deleted"), {'stack': stack_id}) + return + else: + time.sleep(STACK_DELETE_RETRY_WAIT) + stack_delete_retries = stack_delete_retries - 1 + if stack_delete_retries == 0: + LOG.warn(_("Resource cleanup for service chain instance is" + " not completed within %(wait)s seconds as " + "deletion of Stack %(stack)s is not completed"), + {'wait': (STACK_DELETE_RETRIES * + STACK_DELETE_RETRY_WAIT), + 'stack': stack_id}) + return + else: + continue + def _get_instance_by_spec_id(self, context, spec_id): filters = {'servicechain_spec': [spec_id]} return context._plugin.get_servicechain_instances( @@ -332,5 +385,8 @@ class HeatClient: fields['parameters'] = parameters return self.stacks.create(**fields) - def delete(self, id): - return self.stacks.delete(id) + def delete(self, stack_id): + return self.stacks.delete(stack_id) + + def get(self, stack_id): + return self.stacks.get(stack_id) diff --git a/gbpservice/neutron/tests/unit/services/servicechain/test_simple_chain_driver.py b/gbpservice/neutron/tests/unit/services/servicechain/test_simple_chain_driver.py index 15808104f..5ce231cb7 100644 --- a/gbpservice/neutron/tests/unit/services/servicechain/test_simple_chain_driver.py +++ b/gbpservice/neutron/tests/unit/services/servicechain/test_simple_chain_driver.py @@ -24,6 +24,14 @@ import gbpservice.neutron.services.servicechain.drivers.simplechain_driver as\ from gbpservice.neutron.tests.unit.services.servicechain import \ test_servicechain_plugin +STACK_DELETE_RETRIES = 5 +STACK_DELETE_RETRY_WAIT = 3 + + +class MockStackObject(object): + def __init__(self, status): + self.stack_status = status + class SimpleChainDriverTestCase( test_servicechain_plugin.ServiceChainPluginTestCase): @@ -32,6 +40,12 @@ class SimpleChainDriverTestCase( config.cfg.CONF.set_override('servicechain_drivers', ['simplechain_driver'], group='servicechain') + config.cfg.CONF.set_override('stack_delete_retries', + STACK_DELETE_RETRIES, + group='servicechain') + config.cfg.CONF.set_override('stack_delete_retry_wait', + STACK_DELETE_RETRY_WAIT, + group='servicechain') super(SimpleChainDriverTestCase, self).setUp() @@ -163,3 +177,59 @@ class TestServiceChainInstance(SimpleChainDriverTestCase): sc_instance['servicechain_instance']['id']) res = req.get_response(self.ext_api) self.assertEqual(res.status_int, webob.exc.HTTPNoContent.code) + + def test_wait_stack_delete_for_instance_delete(self): + name = "scs1" + scn = self.create_servicechain_node() + scn_id = scn['servicechain_node']['id'] + scs = self.create_servicechain_spec(name=name, nodes=[scn_id]) + sc_spec_id = scs['servicechain_spec']['id'] + + with mock.patch.object(simplechain_driver.HeatClient, + 'create') as stack_create: + stack_create.return_value = {'stack': { + 'id': uuidutils.generate_uuid()}} + sc_instance = self.create_servicechain_instance( + name="sc_instance_1", + servicechain_specs=[sc_spec_id]) + self.assertEqual( + sc_instance['servicechain_instance']['servicechain_specs'], + [sc_spec_id]) + + # Verify that as part of delete service chain instance we call + # get method for heat stack 5 times before giving up if the state + # does not become DELETE_COMPLETE + with contextlib.nested( + mock.patch.object(simplechain_driver.HeatClient, 'delete'), + mock.patch.object(simplechain_driver.HeatClient, 'get')) as ( + stack_delete, stack_get): + stack_get.return_value = MockStackObject('PENDING_DELETE') + req = self.new_delete_request( + 'servicechain_instances', + sc_instance['servicechain_instance']['id']) + res = req.get_response(self.ext_api) + self.assertEqual(res.status_int, webob.exc.HTTPNoContent.code) + stack_delete.assert_called_once_with(mock.ANY) + self.assertEqual(stack_get.call_count, STACK_DELETE_RETRIES) + + # Create and delete another service chain instance and verify that + # we call get method for heat stack only once if the stack state + # is DELETE_COMPLETE + sc_instance = self.create_servicechain_instance( + name="sc_instance_1", + servicechain_specs=[sc_spec_id]) + self.assertEqual( + sc_instance['servicechain_instance']['servicechain_specs'], + [sc_spec_id]) + with contextlib.nested( + mock.patch.object(simplechain_driver.HeatClient, 'delete'), + mock.patch.object(simplechain_driver.HeatClient, 'get')) as ( + stack_delete, stack_get): + stack_get.return_value = MockStackObject('DELETE_COMPLETE') + req = self.new_delete_request( + 'servicechain_instances', + sc_instance['servicechain_instance']['id']) + res = req.get_response(self.ext_api) + self.assertEqual(res.status_int, webob.exc.HTTPNoContent.code) + stack_delete.assert_called_once_with(mock.ANY) + self.assertEqual(stack_get.call_count, 1)