openstack
/
heat
Code Issues Proposed changes

Add resource.root_stack_id column 

This change adds a root_stack_id column to the resource
record to allow a subsequent change enforce
max_resources_per_stack with a single query instead of the
many it currently requires.

This change includes the following:
- Data migration to add the resource.root_stack_id column
  and populate all existing resources with their calculated
  root stack
- Make new resources aquire and set their root_stack_id on
  store or update.
- StackResource._validate_nested_resources use the stored
  root_stack_id resulting in a ~15% performance improvement
  for the creation time of a test stack containing 40 nested
  stacks.

Change-Id: I2b00285514235834131222012408d2b5b2b37d30
Partial-Bug: 1489548
This commit is contained in:
Steve Baker 2015-09-21 16:18:57 +12:00
parent e84f7e4661
commit 1b2cd7495d
7 changed files with 121 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
heat/db/sqlalchemy/migrate_repo/versions/065_root_resource.py Normal file
View File

@ -0,0 +1,48 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy
def upgrade(migrate_engine):
meta = sqlalchemy.MetaData(bind=migrate_engine)
res_table = sqlalchemy.Table('resource', meta, autoload=True)
stack_table = sqlalchemy.Table('stack', meta, autoload=True)
root_stack_id = sqlalchemy.Column('root_stack_id',
sqlalchemy.String(36))
root_stack_id.create(res_table)
root_stack_idx = sqlalchemy.Index('ix_resource_root_stack_id',
res_table.c.root_stack_id,
mysql_length=36)
root_stack_idx.create(migrate_engine)
# build stack->owner relationship for all stacks
stmt = sqlalchemy.select([stack_table.c.id, stack_table.c.owner_id])
stacks = migrate_engine.execute(stmt)
parent_stacks = dict([(s.id, s.owner_id) for s in stacks])
def root_for_stack(stack_id):
owner_id = parent_stacks.get(stack_id)
if owner_id:
return root_for_stack(owner_id)
return stack_id
# for each stack, update the resources with the root_stack_id
for stack_id, owner_id in parent_stacks.items():
root_id = root_for_stack(stack_id)
values = {'root_stack_id': root_id}
update = res_table.update().where(
res_table.c.stack_id == stack_id).values(values)
migrate_engine.execute(update)

1
heat/db/sqlalchemy/models.py
View File

@ -287,6 +287,7 @@ class Resource(BASE, HeatBase, StateAware):
sqlalchemy.ForeignKey('stack.id'),
nullable=False)
stack = relationship(Stack, backref=backref('resources'))
root_stack_id = sqlalchemy.Column(sqlalchemy.String(36), index=True)
data = relationship(ResourceData,
cascade="all,delete",
backref=backref('resource'))

13
heat/engine/resource.py
View File

@ -204,6 +204,7 @@ class Resource(object):
self.replaces = None
self.replaced_by = None
self.current_template_id = None
self.root_stack_id = None
if not stack.has_cache_data(name):
resource = stack.db_resource_get(name)
@ -243,6 +244,7 @@ class Resource(object):
self.replaces = resource.replaces
self.replaced_by = resource.replaced_by
self.current_template_id = resource.current_template_id
self.root_stack_id = resource.root_stack_id
@property
def stack(self):
@ -295,7 +297,8 @@ class Resource(object):
'action': self.INIT,
'status': self.COMPLETE,
'current_template_id': new_tmpl_id,
'stack_name': self.stack.name}
'stack_name': self.stack.name,
'root_stack_id': self.root_stack_id}
new_rs = resource_objects.Resource.create(self.context, rs)
# 2. update the current resource to be replaced_by the one above.
@ -1268,6 +1271,8 @@ class Resource(object):
properties_data_encrypted, properties_data = \
resource_objects.Resource.encrypt_properties_data(
self._stored_properties_data)
if not self.root_stack_id:
self.root_stack_id = self.stack.root_stack_id()
try:
rs = {'action': self.action,
'status': self.status,
@ -1283,7 +1288,8 @@ class Resource(object):
'replaces': self.replaces,
'replaced_by': self.replaced_by,
'current_template_id': self.current_template_id,
'stack_name': self.stack.name}
'stack_name': self.stack.name,
'root_stack_id': self.root_stack_id}
new_rs = resource_objects.Resource.create(self.context, rs)
self.id = new_rs.id
@ -1323,7 +1329,8 @@ class Resource(object):
'replaces': self.replaces,
'replaced_by': self.replaced_by,
'current_template_id': self.current_template_id,
'nova_instance': self.resource_id
'nova_instance': self.resource_id,
'root_stack_id': self.root_stack_id
}
if prev_action == self.INIT:
metadata = self.t.metadata()

3
heat/engine/resources/stack_resource.py
View File

@ -248,9 +248,8 @@ class StackResource(resource.Resource):
def _validate_nested_resources(self, templ):
if cfg.CONF.max_resources_per_stack == -1:
return
root_stack_id = self.stack.root_stack_id()
total_resources = (len(templ[templ.RESOURCES]) +
self.stack.total_resources(root_stack_id))
self.stack.total_resources(self.root_stack_id))
if self.nested():
# It's an update and these resources will be deleted

1
heat/objects/resource.py
View File

@ -64,6 +64,7 @@ class Resource(
'requires': heat_fields.ListField(nullable=True, default=None),
'replaces': fields.IntegerField(nullable=True),
'replaced_by': fields.IntegerField(nullable=True),
'root_stack_id': fields.StringField(nullable=True),
}
@staticmethod

54
heat/tests/db/test_migrations.py
View File

@ -619,6 +619,60 @@ class HeatMigrationsCheckers(test_migrations.WalkVersionsMixin,
self.assertColumnNotExists(engine, 'raw_template',
'predecessor')
def _pre_upgrade_065(self, engine):
raw_template = utils.get_table(engine, 'raw_template')
templ = []
for i in range(960, 963, 1):
t = dict(id=i, template='{}', files='{}')
engine.execute(raw_template.insert(), [t])
templ.append(t)
user_creds = utils.get_table(engine, 'user_creds')
user = [dict(id=uid, username='test_user', password='password',
tenant='test_project', auth_url='bla',
tenant_id=str(uuid.uuid4()),
trust_id='',
trustor_user_id='') for uid in range(960, 963)]
engine.execute(user_creds.insert(), user)
stack = utils.get_table(engine, 'stack')
root_sid = '9a6a3ddb-2219-452c-8fec-a4977f8fe474'
stack_ids = [(root_sid, 0, None),
('b6a23bc2-cd4e-496f-be2e-c11d06124ea2', 1, root_sid),
('7a927947-e004-4afa-8d11-62c1e049ecbd', 2, root_sid)]
data = [dict(id=ll_id, name=ll_id,
owner_id=owner_id,
raw_template_id=templ[templ_id]['id'],
user_creds_id=user[templ_id]['id'],
username='test_user',
disable_rollback=True,
parameters='test_params',
created_at=datetime.datetime.utcnow(),
deleted_at=None)
for ll_id, templ_id, owner_id in stack_ids]
engine.execute(stack.insert(), data)
res_table = utils.get_table(engine, 'resource')
resource_ids = [(960, root_sid),
(961, 'b6a23bc2-cd4e-496f-be2e-c11d06124ea2'),
(962, '7a927947-e004-4afa-8d11-62c1e049ecbd')]
resources = [dict(id=rid, stack_id=sid)
for rid, sid in resource_ids]
engine.execute(res_table.insert(), resources)
def _check_065(self, engine, data):
self.assertColumnExists(engine, 'resource', 'root_stack_id')
res_table = utils.get_table(engine, 'resource')
res_in_db = list(res_table.select().execute())
self.assertTrue(len(res_in_db) >= 3)
# confirm the resource.root_stack_id is set for all resources
for r in res_in_db:
self.assertTrue(r.root_stack_id is not None)
if r.id >= 960 and r.id <= 962:
root_stack_id = '9a6a3ddb-2219-452c-8fec-a4977f8fe474'
self.assertEqual(root_stack_id, r.root_stack_id)
class TestHeatMigrationsMySQL(HeatMigrationsCheckers,
test_base.MySQLOpportunisticTestCase):

16
heat/tests/test_nested_stack.py
View File

@ -84,9 +84,8 @@ Outputs:
stack.store()
return stack
@mock.patch.object(parser.Stack, 'root_stack_id')
@mock.patch.object(parser.Stack, 'total_resources')
def test_nested_stack_three_deep(self, tr, rsi):
def test_nested_stack_three_deep(self, tr):
root_template = '''
HeatTemplateFormatVersion: 2012-12-12
Resources:
@ -118,7 +117,6 @@ Resources:
depth2_template,
self.nested_template]
rsi.return_value = '1234'
tr.return_value = 2
self.validate_stack(root_template)
@ -126,11 +124,9 @@ Resources:
mock.call('https://server.test/depth2.template'),
mock.call('https://server.test/depth3.template')]
urlfetch.get.assert_has_calls(calls)
tr.assert_called_with('1234')
@mock.patch.object(parser.Stack, 'root_stack_id')
@mock.patch.object(parser.Stack, 'total_resources')
def test_nested_stack_six_deep(self, tr, rsi):
def test_nested_stack_six_deep(self, tr):
tmpl = '''
HeatTemplateFormatVersion: 2012-12-12
Resources:
@ -158,11 +154,12 @@ Resources:
depth5_template,
self.nested_template]
rsi.return_value = '1234'
tr.return_value = 5
t = template_format.parse(root_template)
stack = self.parse_stack(t)
stack['Nested'].root_stack_id = '1234'
res = self.assertRaises(exception.StackValidationFailed,
stack.validate)
self.assertIn('Recursion depth exceeds', six.text_type(res))
@ -213,9 +210,8 @@ Resources:
mock.call('https://server.test/depth4.template')]
urlfetch.get.assert_has_calls(calls, any_order=True)
@mock.patch.object(parser.Stack, 'root_stack_id')
@mock.patch.object(parser.Stack, 'total_resources')
def test_nested_stack_infinite_recursion(self, tr, rsi):
def test_nested_stack_infinite_recursion(self, tr):
tmpl = '''
HeatTemplateFormatVersion: 2012-12-12
Resources:
@ -227,7 +223,7 @@ Resources:
urlfetch.get.return_value = tmpl
t = template_format.parse(tmpl)
stack = self.parse_stack(t)
rsi.return_value = '1234'
stack['Nested'].root_stack_id = '1234'
tr.return_value = 2
res = self.assertRaises(exception.StackValidationFailed,
stack.validate)