From 7977f9f2f324f8916433763132c6ee67213e6ed1 Mon Sep 17 00:00:00 2001 From: Thomas Herve Date: Wed, 13 Apr 2016 14:38:59 +0200 Subject: [PATCH] Add command to reset one stack status Adds a new heat-manage reset_stack_status to recover from specific crashes that leaves resources in progress. It removes resource hooks and stack locks as well. Closes-Bug: #1561214 Change-Id: I70fa5857c959bc5f1424d562ff8b7740331b5328 --- heat/cmd/manage.py | 17 ++++++++++ heat/db/api.py | 4 +++ heat/db/sqlalchemy/api.py | 41 +++++++++++++++++++++++ heat/tests/db/test_sqlalchemy_api.py | 49 ++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+) diff --git a/heat/cmd/manage.py b/heat/cmd/manage.py index b73fabb70f..086a8a0c69 100644 --- a/heat/cmd/manage.py +++ b/heat/cmd/manage.py @@ -98,6 +98,18 @@ def do_resource_data_list(): print(print_format % (k, data[k])) +def do_reset_stack_status(): + print(_("Warning: this command is potentially destructive and only " + "intended to recover from specific crashes.")) + print(_("It is advised to shutdown all Heat engines beforehand.")) + print(_("Continue ? [y/N]")) + data = raw_input() + if not data.lower().startswith('y'): + return + ctxt = context.get_admin_context() + db_api.reset_stack_status(ctxt, CONF.command.stack_id) + + def purge_deleted(): """Remove database records that have been previously soft deleted.""" utils.purge_deleted(CONF.command.age, CONF.command.granularity) @@ -157,6 +169,11 @@ def add_command_parsers(subparsers): parser.add_argument('resource_id', help=_('Stack resource id')) + parser = subparsers.add_parser('reset_stack_status') + parser.set_defaults(func=do_reset_stack_status) + parser.add_argument('stack_id', + help=_('Stack id')) + ServiceManageCommand.add_service_parsers(subparsers) command_opt = cfg.SubCommandOpt('command', diff --git a/heat/db/api.py b/heat/db/api.py index 677b930776..862f43811b 100644 --- a/heat/db/api.py +++ b/heat/db/api.py @@ -425,3 +425,7 @@ def db_sync(engine, version=None): def db_version(engine): """Display the current database version.""" return IMPL.db_version(engine) + + +def reset_stack_status(context, stack_id): + return IMPL.reset_stack_status(context, stack_id) diff --git a/heat/db/sqlalchemy/api.py b/heat/db/sqlalchemy/api.py index ad102502a2..773beac52c 100644 --- a/heat/db/sqlalchemy/api.py +++ b/heat/db/sqlalchemy/api.py @@ -39,6 +39,7 @@ from heat.db.sqlalchemy import filters as db_filters from heat.db.sqlalchemy import migration from heat.db.sqlalchemy import models from heat.db.sqlalchemy import utils as db_utils +from heat.engine import environment as heat_environment from heat.rpc import api as rpc_api CONF = cfg.CONF @@ -1385,3 +1386,43 @@ def _get_batch(session, ctxt, query, model, batch_size=50): for result in results: yield result last_batch_marker = results[-1].id + + +def reset_stack_status(context, stack_id, stack=None): + if stack is None: + stack = model_query(context, models.Stack).get(stack_id) + + if stack is None: + raise exception.NotFound(_('Stack with id %s not found') % stack_id) + + session = _session(context) + with session.begin(): + query = model_query(context, models.Resource).filter_by( + status='IN_PROGRESS', stack_id=stack_id) + query.update({'status': 'FAILED', + 'status_reason': 'Stack status manually reset'}) + + query = model_query(context, models.ResourceData) + query = query.join(models.Resource) + query = query.filter_by(stack_id=stack_id) + query = query.filter( + models.ResourceData.key.in_(heat_environment.HOOK_TYPES)) + data_ids = [data.id for data in query] + + if data_ids: + query = model_query(context, models.ResourceData) + query = query.filter(models.ResourceData.id.in_(data_ids)) + query.delete(synchronize_session='fetch') + + query = model_query(context, models.Stack).filter_by(owner_id=stack_id) + for child in query: + reset_stack_status(context, child.id, child) + + with session.begin(): + if stack.status == 'IN_PROGRESS': + stack.status = 'FAILED' + stack.status_reason = 'Stack status manually reset' + + session.query( + models.StackLock + ).filter_by(stack_id=stack_id).delete() diff --git a/heat/tests/db/test_sqlalchemy_api.py b/heat/tests/db/test_sqlalchemy_api.py index 4db919ddc6..6ca151fcfc 100644 --- a/heat/tests/db/test_sqlalchemy_api.py +++ b/heat/tests/db/test_sqlalchemy_api.py @@ -3249,3 +3249,52 @@ class DBAPICryptParamsPropsTest(common.HeatTestCase): self.assertNotEqual(enc_params['param3'], dec_params['param3']) self.assertEqual('bar', dec_params['param2']) self.assertEqual('12345', dec_params['param3']) + + +class ResetStackStatusTests(common.HeatTestCase): + + def setUp(self): + super(ResetStackStatusTests, self).setUp() + self.ctx = utils.dummy_context() + self.template = create_raw_template(self.ctx) + self.user_creds = create_user_creds(self.ctx) + self.stack = create_stack(self.ctx, self.template, self.user_creds) + + def test_status_reset(self): + db_api.stack_update(self.ctx, self.stack.id, {'status': 'IN_PROGRESS'}) + db_api.stack_lock_create(self.stack.id, UUID1) + db_api.reset_stack_status(self.ctx, self.stack.id) + self.assertEqual('FAILED', self.stack.status) + self.assertEqual('Stack status manually reset', + self.stack.status_reason) + self.assertEqual(True, db_api.stack_lock_release(self.stack.id, UUID1)) + + def test_resource_reset(self): + resource_progress = create_resource(self.ctx, self.stack, + status='IN_PROGRESS') + resource_complete = create_resource(self.ctx, self.stack) + db_api.reset_stack_status(self.ctx, self.stack.id) + self.assertEqual('complete', resource_complete.status) + self.assertEqual('FAILED', resource_progress.status) + + def test_hook_reset(self): + resource = create_resource(self.ctx, self.stack) + resource.context = self.ctx + create_resource_data(self.ctx, resource, key="pre-create") + create_resource_data(self.ctx, resource) + db_api.reset_stack_status(self.ctx, self.stack.id) + + vals = db_api.resource_data_get_all(self.ctx, resource.id) + self.assertEqual({'test_resource_key': 'test_value'}, vals) + + def test_nested_stack(self): + db_api.stack_update(self.ctx, self.stack.id, {'status': 'IN_PROGRESS'}) + child = create_stack(self.ctx, self.template, self.user_creds, + owner_id=self.stack.id) + grandchild = create_stack(self.ctx, self.template, self.user_creds, + owner_id=child.id, status='IN_PROGRESS') + resource = create_resource(self.ctx, grandchild, status='IN_PROGRESS') + db_api.reset_stack_status(self.ctx, self.stack.id) + self.assertEqual('FAILED', grandchild.status) + self.assertEqual('FAILED', resource.status) + self.assertEqual('FAILED', self.stack.status)