openstack
/
heat
Code Issues Proposed changes

Purge deleted stacks for specific project 

Add project-id argument to heat-manage purge_deleted command in order
to be able to hard delete DB entries for a specific project.

Change-Id: Ifffe5657a40ce97db9d059ff1516b8e1eb801132
Implements: bp heat-manage-purge-deleted-tenant
This commit is contained in:
Ala Rezmerita 2016-06-30 10:31:54 +02:00
parent 6bd01b350e
commit 61836dbf42
6 changed files with 174 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/source/man/heat-manage.rst
View File

@ -37,9 +37,10 @@ Heat Db version
Sync the database up to the most recent version.
``heat-manage purge_deleted [-g {days,hours,minutes,seconds}] [age]``
``heat-manage purge_deleted [-g {days,hours,minutes,seconds}] [-p project_id] [age]``
Purge db entries marked as deleted and older than [age].
Purge db entries marked as deleted and older than [age]. When project_id
argument is provided, only entries belonging to this project will be purged.
``heat-manage service list``

9
heat/cmd/manage.py
View File

@ -113,7 +113,9 @@ def do_reset_stack_status():
def purge_deleted():
"""Remove database records that have been previously soft deleted."""
utils.purge_deleted(CONF.command.age, CONF.command.granularity)
utils.purge_deleted(CONF.command.age,
CONF.command.granularity,
CONF.command.project_id)
def do_crypt_parameters_and_properties():
@ -150,7 +152,10 @@ def add_command_parsers(subparsers):
'-g', '--granularity', default='days',
choices=['days', 'hours', 'minutes', 'seconds'],
help=_('Granularity to use for age argument, defaults to days.'))
# optional parameter, can be skipped.
parser.add_argument(
'-p', '--project-id',
help=_('Project ID to purge deleted stacks.'))
# update_params parser
parser = subparsers.add_parser('update_params')
parser.set_defaults(func=do_crypt_parameters_and_properties)

21
heat/db/sqlalchemy/api.py
View File

@ -26,6 +26,7 @@ from oslo_utils import timeutils
import osprofiler.sqlalchemy
import six
import sqlalchemy
from sqlalchemy import and_
from sqlalchemy import func
from sqlalchemy import orm
from sqlalchemy.orm import aliased as orm_aliased
@ -1157,7 +1158,7 @@ def service_get_all_by_args(context, host, binary, hostname):
filter_by(hostname=hostname).all())
def purge_deleted(age, granularity='days'):
def purge_deleted(age, granularity='days', project_id=None):
try:
age = int(age)
except ValueError:
@ -1195,10 +1196,20 @@ def purge_deleted(age, granularity='days'):
syncpoint = sqlalchemy.Table('sync_point', meta, autoload=True)
# find the soft-deleted stacks that are past their expiry
stack_where = sqlalchemy.select([stack.c.id, stack.c.raw_template_id,
stack.c.prev_raw_template_id,
stack.c.user_creds_id]).where(
stack.c.deleted_at < time_line)
if project_id:
stack_where = sqlalchemy.select([
stack.c.id, stack.c.raw_template_id,
stack.c.prev_raw_template_id,
stack.c.user_creds_id]).where(and_(
stack.c.tenant == project_id,
stack.c.deleted_at < time_line))
else:
stack_where = sqlalchemy.select([
stack.c.id, stack.c.raw_template_id,
stack.c.prev_raw_template_id,
stack.c.user_creds_id]).where(
stack.c.deleted_at < time_line)
stacks = list(engine.execute(stack_where))
if stacks:
stack_ids = [i[0] for i in stacks]

4
heat/db/utils.py
View File

@ -43,8 +43,8 @@ IMPL = LazyPluggable('backend',
sqlalchemy='heat.db.sqlalchemy.api')
def purge_deleted(age, granularity='days'):
IMPL.purge_deleted(age, granularity)
def purge_deleted(age, granularity='days', project_id=None):
IMPL.purge_deleted(age, granularity, project_id)
def encrypt_parameters_and_properties(ctxt, encryption_key, verbose):

111
heat/tests/db/test_sqlalchemy_api.py
View File

@ -1997,6 +1997,52 @@ class DBAPIStackTest(common.HeatTestCase):
self._deleted_stack_existance(utils.dummy_context(), stacks,
tmpl_files, (), (0, 1, 2, 3, 4))
def test_purge_project_deleted(self):
now = timeutils.utcnow()
delta = datetime.timedelta(seconds=3600 * 7)
deleted = [now - delta * i for i in range(1, 6)]
tmpl_files = [template_files.TemplateFiles(
{'foo': 'file contents %d' % i}) for i in range(5)]
[tmpl_file.store(self.ctx) for tmpl_file in tmpl_files]
templates = [create_raw_template(self.ctx,
files_id=tmpl_files[i].files_id
) for i in range(5)]
values = [
{'tenant': UUID1},
{'tenant': UUID1},
{'tenant': UUID1},
{'tenant': UUID2},
{'tenant': UUID2},
]
creds = [create_user_creds(self.ctx) for i in range(5)]
stacks = [create_stack(self.ctx, templates[i], creds[i],
deleted_at=deleted[i], **values[i]
) for i in range(5)]
db_api.purge_deleted(age=1, granularity='days', project_id=UUID1)
self._deleted_stack_existance(utils.dummy_context(), stacks,
tmpl_files, (0, 1, 2, 3, 4), ())
db_api.purge_deleted(age=22, granularity='hours', project_id=UUID1)
self._deleted_stack_existance(utils.dummy_context(), stacks,
tmpl_files, (0, 1, 2, 3, 4), ())
db_api.purge_deleted(age=1100, granularity='minutes', project_id=UUID1)
self._deleted_stack_existance(utils.dummy_context(), stacks,
tmpl_files, (0, 1, 3, 4), (2,))
db_api.purge_deleted(age=30, granularity='hours', project_id=UUID2)
self._deleted_stack_existance(utils.dummy_context(), stacks,
tmpl_files, (0, 1, 3), (2, 4))
db_api.purge_deleted(age=3600, granularity='seconds', project_id=UUID1)
self._deleted_stack_existance(utils.dummy_context(), stacks,
tmpl_files, (3,), (0, 1, 2, 4))
db_api.purge_deleted(age=3600, granularity='seconds', project_id=UUID2)
self._deleted_stack_existance(utils.dummy_context(), stacks,
tmpl_files, (), (0, 1, 2, 3, 4))
def test_purge_deleted_prev_raw_template(self):
now = timeutils.utcnow()
templates = [create_raw_template(self.ctx) for i in range(2)]
@ -2011,6 +2057,24 @@ class DBAPIStackTest(common.HeatTestCase):
show_deleted=True))
self.assertIsNotNone(db_api.raw_template_get(ctx, templates[1].id))
stacks = [create_stack(self.ctx, templates[0],
create_user_creds(self.ctx),
deleted_at=now - datetime.timedelta(seconds=10),
prev_raw_template=templates[1],
tenant=UUID1)]
db_api.purge_deleted(age=3600, granularity='seconds', project_id=UUID1)
self.assertIsNotNone(db_api.stack_get(ctx, stacks[0].id,
show_deleted=True,
tenant_safe=False))
self.assertIsNotNone(db_api.raw_template_get(ctx, templates[1].id))
db_api.purge_deleted(age=0, granularity='seconds', project_id=UUID2)
self.assertIsNotNone(db_api.stack_get(ctx, stacks[0].id,
show_deleted=True,
tenant_safe=False))
self.assertIsNotNone(db_api.raw_template_get(ctx, templates[1].id))
def test_dont_purge_shared_raw_template_files(self):
now = timeutils.utcnow()
delta = datetime.timedelta(seconds=3600 * 7)
@ -2040,18 +2104,52 @@ class DBAPIStackTest(common.HeatTestCase):
db_api.raw_template_files_get,
self.ctx, tmpl_files[2].files_id)
def test_dont_purge_project_shared_raw_template_files(self):
now = timeutils.utcnow()
delta = datetime.timedelta(seconds=3600 * 7)
deleted = [now - delta * i for i in range(1, 6)]
# the last two template_files are identical to first two
# (so should not be purged)
tmpl_files = [template_files.TemplateFiles(
{'foo': 'more file contents'}) for i in range(3)]
[tmpl_file.store(self.ctx) for tmpl_file in tmpl_files]
templates = [create_raw_template(self.ctx,
files_id=tmpl_files[i % 3].files_id
) for i in range(5)]
creds = [create_user_creds(self.ctx) for i in range(5)]
[create_stack(self.ctx, templates[i], creds[i],
deleted_at=deleted[i], tenant=UUID1
) for i in range(5)]
db_api.purge_deleted(age=0, granularity='seconds', project_id=UUID3)
self.assertIsNotNone(db_api.raw_template_files_get(
self.ctx, tmpl_files[0].files_id))
self.assertIsNotNone(db_api.raw_template_files_get(
self.ctx, tmpl_files[1].files_id))
self.assertIsNotNone(db_api.raw_template_files_get(
self.ctx, tmpl_files[2].files_id))
db_api.purge_deleted(age=15, granularity='hours', project_id=UUID1)
self.assertIsNotNone(db_api.raw_template_files_get(
self.ctx, tmpl_files[0].files_id))
self.assertIsNotNone(db_api.raw_template_files_get(
self.ctx, tmpl_files[1].files_id))
self.assertRaises(exception.NotFound,
db_api.raw_template_files_get,
self.ctx, tmpl_files[2].files_id)
def _deleted_stack_existance(self, ctx, stacks,
tmpl_files, existing, deleted):
tmpl_idx = 0
for s in existing:
self.assertIsNotNone(db_api.stack_get(ctx, stacks[s].id,
show_deleted=True))
show_deleted=True,
tenant_safe=False))
self.assertIsNotNone(db_api.raw_template_files_get(
ctx, tmpl_files[tmpl_idx].files_id))
tmpl_idx = tmpl_idx + 1
ctx, tmpl_files[s].files_id))
for s in deleted:
self.assertIsNone(db_api.stack_get(ctx, stacks[s].id,
show_deleted=True))
show_deleted=True,
tenant_safe=False))
rt_id = stacks[s].raw_template_id
self.assertRaises(exception.NotFound,
db_api.raw_template_get, ctx, rt_id)
@ -2059,13 +2157,12 @@ class DBAPIStackTest(common.HeatTestCase):
ctx, stacks[s].id))
self.assertRaises(exception.NotFound,
db_api.raw_template_files_get,
ctx, tmpl_files[tmpl_idx].files_id)
ctx, tmpl_files[s].files_id)
self.assertEqual([],
db_api.event_get_all_by_stack(ctx,
stacks[s].id))
self.assertIsNone(db_api.user_creds_get(
self.ctx, stacks[s].user_creds_id))
tmpl_idx = tmpl_idx + 1
def test_stack_get_root_id(self):
root = create_stack(self.ctx, self.template, self.user_creds,

42
heat_integrationtests/functional/test_purge.py
View File

@ -11,6 +11,7 @@
# under the License.
import time
import uuid
from oslo_concurrency import processutils
@ -49,3 +50,44 @@ resources:
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertNotIn(stack_identifier.split('/')[1], stacks)
def test_purge_project_id(self):
stack_identifier = self.stack_create(template=self.template)
self._stack_delete(stack_identifier)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertIn(stack_identifier.split('/')[1], stacks)
fake_project_id = uuid.uuid4().hex
time.sleep(1)
cmd = "heat-manage purge_deleted -p %s 0" % fake_project_id
processutils.execute(cmd, shell=True)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertIn(stack_identifier.split('/')[1], stacks)
time.sleep(1)
cmd = "heat-manage purge_deleted 0"
processutils.execute(cmd, shell=True)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertNotIn(stack_identifier.split('/')[1], stacks)
# Test with tags
stack_identifier = self.stack_create(template=self.template,
tags="foo,bar")
self._stack_delete(stack_identifier)
time.sleep(1)
cmd = "heat-manage purge_deleted -p %s 0" % fake_project_id
processutils.execute(cmd, shell=True)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertIn(stack_identifier.split('/')[1], stacks)
time.sleep(1)
cmd = "heat-manage purge_deleted 0"
processutils.execute(cmd, shell=True)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertNotIn(stack_identifier.split('/')[1], stacks)