diff --git a/doc/source/man/heat-manage.rst b/doc/source/man/heat-manage.rst index 88d4d3d13c..ee036385f8 100644 --- a/doc/source/man/heat-manage.rst +++ b/doc/source/man/heat-manage.rst @@ -37,9 +37,10 @@ Heat Db version Sync the database up to the most recent version. -``heat-manage purge_deleted [-g {days,hours,minutes,seconds}] [age]`` +``heat-manage purge_deleted [-g {days,hours,minutes,seconds}] [-p project_id] [age]`` - Purge db entries marked as deleted and older than [age]. + Purge db entries marked as deleted and older than [age]. When project_id + argument is provided, only entries belonging to this project will be purged. ``heat-manage service list`` diff --git a/heat/cmd/manage.py b/heat/cmd/manage.py index 1fe5c46a41..01b816c0f8 100644 --- a/heat/cmd/manage.py +++ b/heat/cmd/manage.py @@ -113,7 +113,9 @@ def do_reset_stack_status(): def purge_deleted(): """Remove database records that have been previously soft deleted.""" - utils.purge_deleted(CONF.command.age, CONF.command.granularity) + utils.purge_deleted(CONF.command.age, + CONF.command.granularity, + CONF.command.project_id) def do_crypt_parameters_and_properties(): @@ -150,7 +152,10 @@ def add_command_parsers(subparsers): '-g', '--granularity', default='days', choices=['days', 'hours', 'minutes', 'seconds'], help=_('Granularity to use for age argument, defaults to days.')) - + # optional parameter, can be skipped. + parser.add_argument( + '-p', '--project-id', + help=_('Project ID to purge deleted stacks.')) # update_params parser parser = subparsers.add_parser('update_params') parser.set_defaults(func=do_crypt_parameters_and_properties) diff --git a/heat/db/sqlalchemy/api.py b/heat/db/sqlalchemy/api.py index 6f4e82d7b7..254f4b5b11 100644 --- a/heat/db/sqlalchemy/api.py +++ b/heat/db/sqlalchemy/api.py @@ -26,6 +26,7 @@ from oslo_utils import timeutils import osprofiler.sqlalchemy import six import sqlalchemy +from sqlalchemy import and_ from sqlalchemy import func from sqlalchemy import orm from sqlalchemy.orm import aliased as orm_aliased @@ -1157,7 +1158,7 @@ def service_get_all_by_args(context, host, binary, hostname): filter_by(hostname=hostname).all()) -def purge_deleted(age, granularity='days'): +def purge_deleted(age, granularity='days', project_id=None): try: age = int(age) except ValueError: @@ -1195,10 +1196,20 @@ def purge_deleted(age, granularity='days'): syncpoint = sqlalchemy.Table('sync_point', meta, autoload=True) # find the soft-deleted stacks that are past their expiry - stack_where = sqlalchemy.select([stack.c.id, stack.c.raw_template_id, - stack.c.prev_raw_template_id, - stack.c.user_creds_id]).where( - stack.c.deleted_at < time_line) + if project_id: + stack_where = sqlalchemy.select([ + stack.c.id, stack.c.raw_template_id, + stack.c.prev_raw_template_id, + stack.c.user_creds_id]).where(and_( + stack.c.tenant == project_id, + stack.c.deleted_at < time_line)) + else: + stack_where = sqlalchemy.select([ + stack.c.id, stack.c.raw_template_id, + stack.c.prev_raw_template_id, + stack.c.user_creds_id]).where( + stack.c.deleted_at < time_line) + stacks = list(engine.execute(stack_where)) if stacks: stack_ids = [i[0] for i in stacks] diff --git a/heat/db/utils.py b/heat/db/utils.py index 580d07bccd..537d99ed38 100644 --- a/heat/db/utils.py +++ b/heat/db/utils.py @@ -43,8 +43,8 @@ IMPL = LazyPluggable('backend', sqlalchemy='heat.db.sqlalchemy.api') -def purge_deleted(age, granularity='days'): - IMPL.purge_deleted(age, granularity) +def purge_deleted(age, granularity='days', project_id=None): + IMPL.purge_deleted(age, granularity, project_id) def encrypt_parameters_and_properties(ctxt, encryption_key, verbose): diff --git a/heat/tests/db/test_sqlalchemy_api.py b/heat/tests/db/test_sqlalchemy_api.py index 945ac4ca30..db3d215ea0 100644 --- a/heat/tests/db/test_sqlalchemy_api.py +++ b/heat/tests/db/test_sqlalchemy_api.py @@ -1997,6 +1997,52 @@ class DBAPIStackTest(common.HeatTestCase): self._deleted_stack_existance(utils.dummy_context(), stacks, tmpl_files, (), (0, 1, 2, 3, 4)) + def test_purge_project_deleted(self): + now = timeutils.utcnow() + delta = datetime.timedelta(seconds=3600 * 7) + deleted = [now - delta * i for i in range(1, 6)] + tmpl_files = [template_files.TemplateFiles( + {'foo': 'file contents %d' % i}) for i in range(5)] + [tmpl_file.store(self.ctx) for tmpl_file in tmpl_files] + templates = [create_raw_template(self.ctx, + files_id=tmpl_files[i].files_id + ) for i in range(5)] + values = [ + {'tenant': UUID1}, + {'tenant': UUID1}, + {'tenant': UUID1}, + {'tenant': UUID2}, + {'tenant': UUID2}, + ] + creds = [create_user_creds(self.ctx) for i in range(5)] + stacks = [create_stack(self.ctx, templates[i], creds[i], + deleted_at=deleted[i], **values[i] + ) for i in range(5)] + + db_api.purge_deleted(age=1, granularity='days', project_id=UUID1) + self._deleted_stack_existance(utils.dummy_context(), stacks, + tmpl_files, (0, 1, 2, 3, 4), ()) + + db_api.purge_deleted(age=22, granularity='hours', project_id=UUID1) + self._deleted_stack_existance(utils.dummy_context(), stacks, + tmpl_files, (0, 1, 2, 3, 4), ()) + + db_api.purge_deleted(age=1100, granularity='minutes', project_id=UUID1) + self._deleted_stack_existance(utils.dummy_context(), stacks, + tmpl_files, (0, 1, 3, 4), (2,)) + + db_api.purge_deleted(age=30, granularity='hours', project_id=UUID2) + self._deleted_stack_existance(utils.dummy_context(), stacks, + tmpl_files, (0, 1, 3), (2, 4)) + + db_api.purge_deleted(age=3600, granularity='seconds', project_id=UUID1) + self._deleted_stack_existance(utils.dummy_context(), stacks, + tmpl_files, (3,), (0, 1, 2, 4)) + + db_api.purge_deleted(age=3600, granularity='seconds', project_id=UUID2) + self._deleted_stack_existance(utils.dummy_context(), stacks, + tmpl_files, (), (0, 1, 2, 3, 4)) + def test_purge_deleted_prev_raw_template(self): now = timeutils.utcnow() templates = [create_raw_template(self.ctx) for i in range(2)] @@ -2011,6 +2057,24 @@ class DBAPIStackTest(common.HeatTestCase): show_deleted=True)) self.assertIsNotNone(db_api.raw_template_get(ctx, templates[1].id)) + stacks = [create_stack(self.ctx, templates[0], + create_user_creds(self.ctx), + deleted_at=now - datetime.timedelta(seconds=10), + prev_raw_template=templates[1], + tenant=UUID1)] + + db_api.purge_deleted(age=3600, granularity='seconds', project_id=UUID1) + self.assertIsNotNone(db_api.stack_get(ctx, stacks[0].id, + show_deleted=True, + tenant_safe=False)) + self.assertIsNotNone(db_api.raw_template_get(ctx, templates[1].id)) + + db_api.purge_deleted(age=0, granularity='seconds', project_id=UUID2) + self.assertIsNotNone(db_api.stack_get(ctx, stacks[0].id, + show_deleted=True, + tenant_safe=False)) + self.assertIsNotNone(db_api.raw_template_get(ctx, templates[1].id)) + def test_dont_purge_shared_raw_template_files(self): now = timeutils.utcnow() delta = datetime.timedelta(seconds=3600 * 7) @@ -2040,18 +2104,52 @@ class DBAPIStackTest(common.HeatTestCase): db_api.raw_template_files_get, self.ctx, tmpl_files[2].files_id) + def test_dont_purge_project_shared_raw_template_files(self): + now = timeutils.utcnow() + delta = datetime.timedelta(seconds=3600 * 7) + deleted = [now - delta * i for i in range(1, 6)] + # the last two template_files are identical to first two + # (so should not be purged) + tmpl_files = [template_files.TemplateFiles( + {'foo': 'more file contents'}) for i in range(3)] + [tmpl_file.store(self.ctx) for tmpl_file in tmpl_files] + templates = [create_raw_template(self.ctx, + files_id=tmpl_files[i % 3].files_id + ) for i in range(5)] + creds = [create_user_creds(self.ctx) for i in range(5)] + [create_stack(self.ctx, templates[i], creds[i], + deleted_at=deleted[i], tenant=UUID1 + ) for i in range(5)] + + db_api.purge_deleted(age=0, granularity='seconds', project_id=UUID3) + self.assertIsNotNone(db_api.raw_template_files_get( + self.ctx, tmpl_files[0].files_id)) + self.assertIsNotNone(db_api.raw_template_files_get( + self.ctx, tmpl_files[1].files_id)) + self.assertIsNotNone(db_api.raw_template_files_get( + self.ctx, tmpl_files[2].files_id)) + + db_api.purge_deleted(age=15, granularity='hours', project_id=UUID1) + self.assertIsNotNone(db_api.raw_template_files_get( + self.ctx, tmpl_files[0].files_id)) + self.assertIsNotNone(db_api.raw_template_files_get( + self.ctx, tmpl_files[1].files_id)) + self.assertRaises(exception.NotFound, + db_api.raw_template_files_get, + self.ctx, tmpl_files[2].files_id) + def _deleted_stack_existance(self, ctx, stacks, tmpl_files, existing, deleted): - tmpl_idx = 0 for s in existing: self.assertIsNotNone(db_api.stack_get(ctx, stacks[s].id, - show_deleted=True)) + show_deleted=True, + tenant_safe=False)) self.assertIsNotNone(db_api.raw_template_files_get( - ctx, tmpl_files[tmpl_idx].files_id)) - tmpl_idx = tmpl_idx + 1 + ctx, tmpl_files[s].files_id)) for s in deleted: self.assertIsNone(db_api.stack_get(ctx, stacks[s].id, - show_deleted=True)) + show_deleted=True, + tenant_safe=False)) rt_id = stacks[s].raw_template_id self.assertRaises(exception.NotFound, db_api.raw_template_get, ctx, rt_id) @@ -2059,13 +2157,12 @@ class DBAPIStackTest(common.HeatTestCase): ctx, stacks[s].id)) self.assertRaises(exception.NotFound, db_api.raw_template_files_get, - ctx, tmpl_files[tmpl_idx].files_id) + ctx, tmpl_files[s].files_id) self.assertEqual([], db_api.event_get_all_by_stack(ctx, stacks[s].id)) self.assertIsNone(db_api.user_creds_get( self.ctx, stacks[s].user_creds_id)) - tmpl_idx = tmpl_idx + 1 def test_stack_get_root_id(self): root = create_stack(self.ctx, self.template, self.user_creds, diff --git a/heat_integrationtests/functional/test_purge.py b/heat_integrationtests/functional/test_purge.py index fd652a971c..ef742ef7bb 100644 --- a/heat_integrationtests/functional/test_purge.py +++ b/heat_integrationtests/functional/test_purge.py @@ -11,6 +11,7 @@ # under the License. import time +import uuid from oslo_concurrency import processutils @@ -49,3 +50,44 @@ resources: stacks = dict((stack.id, stack) for stack in self.client.stacks.list(show_deleted=True)) self.assertNotIn(stack_identifier.split('/')[1], stacks) + + def test_purge_project_id(self): + stack_identifier = self.stack_create(template=self.template) + self._stack_delete(stack_identifier) + stacks = dict((stack.id, stack) for stack in + self.client.stacks.list(show_deleted=True)) + self.assertIn(stack_identifier.split('/')[1], stacks) + fake_project_id = uuid.uuid4().hex + time.sleep(1) + cmd = "heat-manage purge_deleted -p %s 0" % fake_project_id + processutils.execute(cmd, shell=True) + + stacks = dict((stack.id, stack) for stack in + self.client.stacks.list(show_deleted=True)) + self.assertIn(stack_identifier.split('/')[1], stacks) + + time.sleep(1) + cmd = "heat-manage purge_deleted 0" + processutils.execute(cmd, shell=True) + stacks = dict((stack.id, stack) for stack in + self.client.stacks.list(show_deleted=True)) + self.assertNotIn(stack_identifier.split('/')[1], stacks) + + # Test with tags + stack_identifier = self.stack_create(template=self.template, + tags="foo,bar") + self._stack_delete(stack_identifier) + + time.sleep(1) + cmd = "heat-manage purge_deleted -p %s 0" % fake_project_id + processutils.execute(cmd, shell=True) + stacks = dict((stack.id, stack) for stack in + self.client.stacks.list(show_deleted=True)) + self.assertIn(stack_identifier.split('/')[1], stacks) + + time.sleep(1) + cmd = "heat-manage purge_deleted 0" + processutils.execute(cmd, shell=True) + stacks = dict((stack.id, stack) for stack in + self.client.stacks.list(show_deleted=True)) + self.assertNotIn(stack_identifier.split('/')[1], stacks)