openstack/heat
Code Issues Proposed changes

Fix database purge query

Browse Source 
Fix raw_template purge query on MySQL, and handle stack tags before
removing stacks. This also removes a bunch of race conditions where we
deleted incorrect data.

Change-Id: I7b7a1d94acefbaeeed86f1833c979819361c8988
Closes-Bug: #1524387
This commit is contained in:
Thomas Herve 2015-12-09 16:45:30 +01:00
parent f84d2ea795
commit 6cece5bb7e
3 changed files with 113 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
heat/db/sqlalchemy/api.py
View File

@ -1064,6 +1064,7 @@ def purge_deleted(age, granularity='days'):
stack = sqlalchemy.Table('stack', meta, autoload=True)
stack_lock = sqlalchemy.Table('stack_lock', meta, autoload=True)
stack_tag = sqlalchemy.Table('stack_tag', meta, autoload=True)
resource = sqlalchemy.Table('resource', meta, autoload=True)
resource_data = sqlalchemy.Table('resource_data', meta, autoload=True)
event = sqlalchemy.Table('event', meta, autoload=True)
@ -1073,47 +1074,68 @@ def purge_deleted(age, granularity='days'):
syncpoint = sqlalchemy.Table('sync_point', meta, autoload=True)
# find the soft-deleted stacks that are past their expiry
stack_where = sqlalchemy.select([stack.c.id]).where(
stack.c.deleted_at < time_line)
# delete stack locks (just in case some got stuck)
stack_lock_del = stack_lock.delete().where(
stack_lock.c.stack_id.in_(stack_where))
engine.execute(stack_lock_del)
# delete resource_data
res_where = sqlalchemy.select([resource.c.id]).where(
resource.c.stack_id.in_(stack_where))
res_data_del = resource_data.delete().where(
resource_data.c.resource_id.in_(res_where))
engine.execute(res_data_del)
# delete resources
res_del = resource.delete().where(resource.c.stack_id.in_(stack_where))
engine.execute(res_del)
# delete events
event_del = event.delete().where(event.c.stack_id.in_(stack_where))
engine.execute(event_del)
# clean up any sync_points that may have lingered
sync_del = syncpoint.delete().where(syncpoint.c.stack_id.in_(stack_where))
engine.execute(sync_del)
# delete the stacks
stack_del = stack.delete().where(stack.c.deleted_at < time_line)
engine.execute(stack_del)
# delete orphaned raw templates
raw_templ_sel = raw_template.c.id.in_(
sqlalchemy.select([raw_template.c.id]).select_from(
sqlalchemy.join(
raw_template,
stack,
sqlalchemy.or_(
stack.c.prev_raw_template_id == raw_template.c.id,
stack.c.raw_template_id == raw_template.c.id),
isouter=True)).where(stack.c.id == None)) # noqa
raw_templ_del = raw_template.delete().where(raw_templ_sel)
engine.execute(raw_templ_del)
# purge any user creds that are no longer referenced
stack_creds_sel = sqlalchemy.select([stack.c.user_creds_id])
user_creds_sel = sqlalchemy.not_(user_creds.c.id.in_(stack_creds_sel))
usr_creds_del = user_creds.delete().where(user_creds_sel)
engine.execute(usr_creds_del)
stack_where = sqlalchemy.select([stack.c.id, stack.c.raw_template_id,
stack.c.prev_raw_template_id,
stack.c.user_creds_id]).where(
stack.c.deleted_at < time_line)
stacks = list(engine.execute(stack_where))
if stacks:
stack_ids = [i[0] for i in stacks]
# delete stack locks (just in case some got stuck)
stack_lock_del = stack_lock.delete().where(
stack_lock.c.stack_id.in_(stack_ids))
engine.execute(stack_lock_del)
# delete stack tags
stack_tag_del = stack_tag.delete().where(
stack_tag.c.stack_id.in_(stack_ids))
engine.execute(stack_tag_del)
# delete resource_data
res_where = sqlalchemy.select([resource.c.id]).where(
resource.c.stack_id.in_(stack_ids))
res_data_del = resource_data.delete().where(
resource_data.c.resource_id.in_(res_where))
engine.execute(res_data_del)
# delete resources
res_del = resource.delete().where(resource.c.stack_id.in_(stack_ids))
engine.execute(res_del)
# delete events
event_del = event.delete().where(event.c.stack_id.in_(stack_ids))
engine.execute(event_del)
# clean up any sync_points that may have lingered
sync_del = syncpoint.delete().where(
syncpoint.c.stack_id.in_(stack_ids))
engine.execute(sync_del)
# delete the stacks
stack_del = stack.delete().where(stack.c.id.in_(stack_ids))
engine.execute(stack_del)
# delete orphaned raw templates
raw_template_ids = [i[1] for i in stacks if i[1] is not None]
raw_template_ids.extend(i[2] for i in stacks if i[2] is not None)
if raw_template_ids:
# keep those still referenced
raw_tmpl_sel = sqlalchemy.select([stack.c.raw_template_id]).where(
stack.c.raw_template_id.in_(raw_template_ids))
raw_tmpl = [i[0] for i in engine.execute(raw_tmpl_sel)]
raw_template_ids = set(raw_template_ids) - set(raw_tmpl)
raw_tmpl_sel = sqlalchemy.select(
[stack.c.prev_raw_template_id]).where(
stack.c.prev_raw_template_id.in_(raw_template_ids))
raw_tmpl = [i[0] for i in engine.execute(raw_tmpl_sel)]
raw_template_ids = raw_template_ids - set(raw_tmpl)
raw_templ_del = raw_template.delete().where(
raw_template.c.id.in_(raw_template_ids))
engine.execute(raw_templ_del)
# purge any user creds that are no longer referenced
user_creds_ids = [i[3] for i in stacks if i[3] is not None]
if user_creds_ids:
# keep those still referenced
user_sel = sqlalchemy.select([stack.c.user_creds_id]).where(
stack.c.user_creds_id.in_(user_creds_ids))
users = [i[0] for i in engine.execute(user_sel)]
user_creds_ids = set(user_creds_ids) - set(users)
usr_creds_del = user_creds.delete().where(
user_creds.c.id.in_(user_creds_ids))
engine.execute(usr_creds_del)
# Purge deleted services
srvc_del = service.delete().where(service.c.deleted_at < time_line)
engine.execute(srvc_del)

3
heat/objects/stack.py
View File

@ -189,6 +189,9 @@ class Stack(
def refresh(self):
db_stack = db_api.stack_get(
self._context, self.id, show_deleted=True)
if db_stack is None:
message = _('No stack exists with id "%s"') % str(self.id)
raise exception.NotFound(message)
db_stack.refresh()
return self.__class__._from_db_object(
self._context,

47
heat_integrationtests/functional/test_purge.py Normal file
View File

@ -0,0 +1,47 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_concurrency import processutils
from heat_integrationtests.functional import functional_base
class PurgeTest(functional_base.FunctionalTestsBase):
template = '''
heat_template_version: 2014-10-16
parameters:
resources:
test_resource:
type: OS::Heat::TestResource
'''
def test_purge(self):
stack_identifier = self.stack_create(template=self.template)
self._stack_delete(stack_identifier)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertIn(stack_identifier.split('/')[1], stacks)
cmd = "heat-manage purge_deleted 0"
processutils.execute(cmd, shell=True)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertNotIn(stack_identifier.split('/')[1], stacks)
# Test with tags
stack_identifier = self.stack_create(template=self.template,
tags="foo,bar")
self._stack_delete(stack_identifier)
cmd = "heat-manage purge_deleted 0"
processutils.execute(cmd, shell=True)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertNotIn(stack_identifier.split('/')[1], stacks)