x/group-based-policy
Code Issues Proposed changes

Wait for resource cleanup in Servicechain Instance Delete

Browse Source 
Change-Id: Iecb99e5f4715d7844762d6e1e39154f41a368ad4
Closes-bug:1406738
This commit is contained in:
Magesh GV
2015-01-02 19:29:24 +05:30
parent 67b74e0aec
commit 08ac2a0d8c
2 changed files with 128 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
gbpservice/neutron/services/servicechain/drivers/simplechain_driver.py
View File

@@ -11,6 +11,8 @@
# under the License.
import ast
import time
from heatclient import client as heat_client
from neutron.common import log
from neutron.db import model_base
@@ -27,8 +29,23 @@ from gbpservice.neutron.services.servicechain.common import exceptions as exc
LOG = logging.getLogger(__name__)
service_chain_opts = [
cfg.IntOpt('stack_delete_retries',
default=5,
help=_("Number of attempts to retry for stack deletion")),
cfg.IntOpt('stack_delete_retry_wait',
default=3,
help=_("Wait time between two successive stack delete "
"retries")),
]
cfg.CONF.register_opts(service_chain_opts, "servicechain")
# Service chain API supported Values
sc_supported_type = [pconst.LOADBALANCER, pconst.FIREWALL]
STACK_DELETE_RETRIES = cfg.CONF.servicechain.stack_delete_retries
STACK_DELETE_RETRY_WAIT = cfg.CONF.servicechain.stack_delete_retry_wait
class ServiceChainInstanceStack(model_base.BASEV2):
@@ -248,8 +265,44 @@ class SimpleChainDriver(object):
heatclient = HeatClient(context)
for stack in stack_ids:
heatclient.delete(stack.stack_id)
for stack in stack_ids:
self._wait_for_stack_delete(heatclient, stack.stack_id)
self._delete_chain_stacks_db(context.session, instance_id)
# Wait for the heat stack to be deleted for a maximum of 15 seconds
# we check the status every 3 seconds and call sleep again
# This is required because cleanup of subnet fails when the stack created
# some ports on the subnet and the resource delete is not completed by
# the time subnet delete is triggered by Resource Mapping driver
def _wait_for_stack_delete(self, heatclient, stack_id):
stack_delete_retries = STACK_DELETE_RETRIES
while True:
try:
stack = heatclient.get(stack_id)
if stack.stack_status == 'DELETE_COMPLETE':
return
elif stack.stack_status == 'ERROR':
heatclient.delete(stack_id)
except Exception:
LOG.exception(_("Service Chain Instance cleanup may not have "
"happened because Heat API request failed "
"while waiting for the stack %(stack)s to be "
"deleted"), {'stack': stack_id})
return
else:
time.sleep(STACK_DELETE_RETRY_WAIT)
stack_delete_retries = stack_delete_retries - 1
if stack_delete_retries == 0:
LOG.warn(_("Resource cleanup for service chain instance is"
" not completed within %(wait)s seconds as "
"deletion of Stack %(stack)s is not completed"),
{'wait': (STACK_DELETE_RETRIES *
STACK_DELETE_RETRY_WAIT),
'stack': stack_id})
return
else:
continue
def _get_instance_by_spec_id(self, context, spec_id):
filters = {'servicechain_spec': [spec_id]}
return context._plugin.get_servicechain_instances(
@@ -332,5 +385,8 @@ class HeatClient:
fields['parameters'] = parameters
return self.stacks.create(**fields)
def delete(self, id):
return self.stacks.delete(id)
def delete(self, stack_id):
return self.stacks.delete(stack_id)
def get(self, stack_id):
return self.stacks.get(stack_id)

70
gbpservice/neutron/tests/unit/services/servicechain/test_simple_chain_driver.py
View File

@@ -24,6 +24,14 @@ import gbpservice.neutron.services.servicechain.drivers.simplechain_driver as\
from gbpservice.neutron.tests.unit.services.servicechain import \
test_servicechain_plugin
STACK_DELETE_RETRIES = 5
STACK_DELETE_RETRY_WAIT = 3
class MockStackObject(object):
def __init__(self, status):
self.stack_status = status
class SimpleChainDriverTestCase(
test_servicechain_plugin.ServiceChainPluginTestCase):
@@ -32,6 +40,12 @@ class SimpleChainDriverTestCase(
config.cfg.CONF.set_override('servicechain_drivers',
['simplechain_driver'],
group='servicechain')
config.cfg.CONF.set_override('stack_delete_retries',
STACK_DELETE_RETRIES,
group='servicechain')
config.cfg.CONF.set_override('stack_delete_retry_wait',
STACK_DELETE_RETRY_WAIT,
group='servicechain')
super(SimpleChainDriverTestCase, self).setUp()
@@ -163,3 +177,59 @@ class TestServiceChainInstance(SimpleChainDriverTestCase):
sc_instance['servicechain_instance']['id'])
res = req.get_response(self.ext_api)
self.assertEqual(res.status_int, webob.exc.HTTPNoContent.code)
def test_wait_stack_delete_for_instance_delete(self):
name = "scs1"
scn = self.create_servicechain_node()
scn_id = scn['servicechain_node']['id']
scs = self.create_servicechain_spec(name=name, nodes=[scn_id])
sc_spec_id = scs['servicechain_spec']['id']
with mock.patch.object(simplechain_driver.HeatClient,
'create') as stack_create:
stack_create.return_value = {'stack': {
'id': uuidutils.generate_uuid()}}
sc_instance = self.create_servicechain_instance(
name="sc_instance_1",
servicechain_specs=[sc_spec_id])
self.assertEqual(
sc_instance['servicechain_instance']['servicechain_specs'],
[sc_spec_id])
# Verify that as part of delete service chain instance we call
# get method for heat stack 5 times before giving up if the state
# does not become DELETE_COMPLETE
with contextlib.nested(
mock.patch.object(simplechain_driver.HeatClient, 'delete'),
mock.patch.object(simplechain_driver.HeatClient, 'get')) as (
stack_delete, stack_get):
stack_get.return_value = MockStackObject('PENDING_DELETE')
req = self.new_delete_request(
'servicechain_instances',
sc_instance['servicechain_instance']['id'])
res = req.get_response(self.ext_api)
self.assertEqual(res.status_int, webob.exc.HTTPNoContent.code)
stack_delete.assert_called_once_with(mock.ANY)
self.assertEqual(stack_get.call_count, STACK_DELETE_RETRIES)
# Create and delete another service chain instance and verify that
# we call get method for heat stack only once if the stack state
# is DELETE_COMPLETE
sc_instance = self.create_servicechain_instance(
name="sc_instance_1",
servicechain_specs=[sc_spec_id])
self.assertEqual(
sc_instance['servicechain_instance']['servicechain_specs'],
[sc_spec_id])
with contextlib.nested(
mock.patch.object(simplechain_driver.HeatClient, 'delete'),
mock.patch.object(simplechain_driver.HeatClient, 'get')) as (
stack_delete, stack_get):
stack_get.return_value = MockStackObject('DELETE_COMPLETE')
req = self.new_delete_request(
'servicechain_instances',
sc_instance['servicechain_instance']['id'])
res = req.get_response(self.ext_api)
self.assertEqual(res.status_int, webob.exc.HTTPNoContent.code)
stack_delete.assert_called_once_with(mock.ANY)
self.assertEqual(stack_get.call_count, 1)