openstack/deb-heat
Code Issues Proposed changes

heat engine : Implement rollback for stack create

Browse Source 
Implement stack rollback for create_stack, so if a stack
creation fails we automatically rollback (ie delete) the
stack

NOTE : this option defaults to on, so use the --disable-rollback
option to the CLI tools if you want to disable this feature

blueprint stack-rollback

Change-Id: I70a3822426706d0787e571517e059baff1406c0f
This commit is contained in:
Steven Hardy
2013-02-14 16:17:25 +00:00
parent 5bf32e30d0
commit 54defea528
7 changed files with 120 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
heat/db/sqlalchemy/migrate_repo/versions/015_add_stack_rollback_col.py Normal file
View File

@@ -0,0 +1,26 @@
from sqlalchemy import *
from migrate import *
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
stack = Table('stack', meta, autoload=True)
# Note hard-coded default 60 (minutes) here from the value in the
# engine, means we can upgrade and populate existing rows
try:
col = Column('disable_rollback', Boolean, nullable=False, default=True)
col.create(stack, populate_default=True)
except Exception as ex:
print "Caught exception adding disable_rollback column %s" % ex
# *Hack-alert* Sqlite in the unit tests can't handle the above
# approach to nullable=False, so retry with nullable=True
Column('disable_rollback', Boolean, nullable=True,
default=60).create(stack)
def downgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
stack = Table('stack', meta, autoload=True)
stack.c.disable_rollback.drop()

1
heat/db/sqlalchemy/models.py
View File

@@ -162,6 +162,7 @@ class Stack(BASE, HeatBase):
nullable=False)
owner_id = Column(Integer, nullable=True)
timeout = Column(Integer)
disable_rollback = Column(Boolean)
class UserCreds(BASE, HeatBase):

45
heat/engine/parser.py
View File

@@ -35,6 +35,10 @@ logger = logging.getLogger(__name__)
class Stack(object):
ACTIONS = (CREATE, DELETE, UPDATE, ROLLBACK
) = ('CREATE', 'DELETE', 'UPDATE', 'ROLLBACK')
CREATE_IN_PROGRESS = 'CREATE_IN_PROGRESS'
CREATE_FAILED = 'CREATE_FAILED'
CREATE_COMPLETE = 'CREATE_COMPLETE'
@@ -47,12 +51,16 @@ class Stack(object):
UPDATE_COMPLETE = 'UPDATE_COMPLETE'
UPDATE_FAILED = 'UPDATE_FAILED'
ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
ROLLBACK_FAILED = 'ROLLBACK_FAILED'
created_time = timestamp.Timestamp(db_api.stack_get, 'created_at')
updated_time = timestamp.Timestamp(db_api.stack_get, 'updated_at')
def __init__(self, context, stack_name, tmpl, parameters=None,
stack_id=None, state=None, state_description='',
timeout_mins=60, resolve_data=True):
timeout_mins=60, resolve_data=True, disable_rollback=False):
'''
Initialise from a context, name, Template object and (optionally)
Parameters object. The database ID may also be initialised, if the
@@ -70,6 +78,7 @@ class Stack(object):
self.state = state
self.state_description = state_description
self.timeout_mins = timeout_mins
self.disable_rollback = disable_rollback
if parameters is None:
parameters = Parameters(self.name, self.t)
@@ -109,7 +118,7 @@ class Stack(object):
params = Parameters(stack.name, template, stack.parameters)
stack = cls(context, stack.name, template, params,
stack.id, stack.status, stack.status_reason, stack.timeout,
resolve_data)
resolve_data, stack.disable_rollback)
return stack
@@ -131,6 +140,7 @@ class Stack(object):
'status': self.state,
'status_reason': self.state_description,
'timeout': self.timeout_mins,
'disable_rollback': self.disable_rollback,
}
if self.id:
db_api.stack_update(self.context, self.id, s)
@@ -251,6 +261,9 @@ class Stack(object):
self.state_set(stack_status, reason)
if stack_status == self.CREATE_FAILED and not self.disable_rollback:
self.delete(action=self.ROLLBACK)
def update(self, newstack):
'''
Compare the current stack with newstack,
@@ -375,11 +388,22 @@ class Stack(object):
self.state_set(stack_status, reason)
def delete(self):
def delete(self, action=DELETE):
'''
Delete all of the resources, and then the stack itself.
The action parameter is used to differentiate between a user
initiated delete and an automatic stack rollback after a failed
create, which amount to the same thing, but the states are recorded
differently.
'''
self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started')
if action == self.DELETE:
self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started')
elif action == self.ROLLBACK:
self.state_set(self.ROLLBACK_IN_PROGRESS, 'Stack rollback started')
else:
logger.error("Unexpected action %s passed to delete!" % action)
self.state_set(self.DELETE_FAILED, "Invalid action %s" % action)
return
failures = []
for res in reversed(self):
@@ -390,10 +414,17 @@ class Stack(object):
failures.append(str(res))
if failures:
self.state_set(self.DELETE_FAILED,
'Failed to delete ' + ', '.join(failures))
if action == self.DELETE:
self.state_set(self.DELETE_FAILED,
'Failed to delete ' + ', '.join(failures))
elif action == self.ROLLBACK:
self.state_set(self.ROLLBACK_FAILED,
'Failed to rollback ' + ', '.join(failures))
else:
self.state_set(self.DELETE_COMPLETE, 'Deleted successfully')
if action == self.DELETE:
self.state_set(self.DELETE_COMPLETE, 'Deleted successfully')
elif action == self.ROLLBACK:
self.state_set(self.ROLLBACK_COMPLETE, 'Rollback completed')
db_api.stack_delete(self.context, self.id)
def output(self, key):

5
heat/engine/stack_resource.py
View File

@@ -53,10 +53,13 @@ class StackResource(resource.Resource):
params = parser.Parameters(self.physical_resource_name(), template,
user_params)
# Note we disable rollback for nested stacks, since they
# should be rolled back by the parent stack on failure
self._nested = parser.Stack(self.context,
self.physical_resource_name(),
template,
params)
params,
disable_rollback=True)
nested_id = self._nested.store(self.stack)
self.resource_id_set(nested_id)

2
heat/tests/test_loadbalancer.py
View File

@@ -69,7 +69,7 @@ class LoadBalancerTest(unittest.TestCase):
template = parser.Template(t)
params = parser.Parameters('test_stack', template, {'KeyName': 'test'})
stack = parser.Stack(create_context(self.m), 'test_stack', template,
params, stack_id=None)
params, stack_id=None, disable_rollback=True)
stack.store()
return stack

46
heat/tests/test_parser.py
View File

@@ -25,6 +25,7 @@ from heat.engine import parameters
from heat.engine import template
from heat.engine.resource import Resource
from heat.tests.utils import stack_delete_after
import heat.db as db_api
def join(raw):
@@ -358,3 +359,48 @@ class StackTest(unittest.TestCase):
self.stack.state_set(self.stack.CREATE_IN_PROGRESS, 'testing')
self.assertNotEqual(self.stack.updated_time, None)
self.assertNotEqual(self.stack.updated_time, stored_time)
@stack_delete_after
def test_delete(self):
self.stack = parser.Stack(self.ctx, 'delete_test',
parser.Template({}))
stack_id = self.stack.store()
db_s = db_api.stack_get(self.ctx, stack_id)
self.assertNotEqual(db_s, None)
self.stack.delete()
db_s = db_api.stack_get(self.ctx, stack_id)
self.assertEqual(db_s, None)
self.assertEqual(self.stack.state, self.stack.DELETE_COMPLETE)
@stack_delete_after
def test_delete_rollback(self):
self.stack = parser.Stack(self.ctx, 'delete_rollback_test',
parser.Template({}))
stack_id = self.stack.store()
db_s = db_api.stack_get(self.ctx, stack_id)
self.assertNotEqual(db_s, None)
self.stack.delete(action=self.stack.ROLLBACK)
db_s = db_api.stack_get(self.ctx, stack_id)
self.assertEqual(db_s, None)
self.assertEqual(self.stack.state, self.stack.ROLLBACK_COMPLETE)
@stack_delete_after
def test_delete_badaction(self):
self.stack = parser.Stack(self.ctx, 'delete_badaction_test',
parser.Template({}))
stack_id = self.stack.store()
db_s = db_api.stack_get(self.ctx, stack_id)
self.assertNotEqual(db_s, None)
self.stack.delete(action="wibble")
db_s = db_api.stack_get(self.ctx, stack_id)
self.assertNotEqual(db_s, None)
self.assertEqual(self.stack.state, self.stack.DELETE_FAILED)

6
heat/tests/test_waitcondition.py
View File

@@ -104,7 +104,8 @@ class WaitConditionTest(unittest.TestCase):
parameters = parser.Parameters(stack_name, template, params)
ctx = context.get_admin_context()
ctx.tenant_id = 'test_tenant'
stack = parser.Stack(ctx, stack_name, template, parameters)
stack = parser.Stack(ctx, stack_name, template, parameters,
disable_rollback=True)
self.stack_id = stack.store()
@@ -396,7 +397,8 @@ class WaitConditionHandleTest(unittest.TestCase):
parameters = parser.Parameters(stack_name, template, params)
ctx = context.get_admin_context()
ctx.tenant_id = 'test_tenant'
stack = parser.Stack(ctx, stack_name, template, parameters)
stack = parser.Stack(ctx, stack_name, template, parameters,
disable_rollback=True)
# Stub out the UUID for this test, so we can get an expected signature
self.m.StubOutWithMock(uuid, 'uuid4')
uuid.uuid4().AndReturn('STACKABCD1234')