Add resource.root_stack_id column
This change adds a root_stack_id column to the resource record to allow a subsequent change enforce max_resources_per_stack with a single query instead of the many it currently requires. This change includes the following: - Data migration to add the resource.root_stack_id column and populate all existing resources with their calculated root stack - Make new resources aquire and set their root_stack_id on store or update. - StackResource._validate_nested_resources use the stored root_stack_id resulting in a ~15% performance improvement for the creation time of a test stack containing 40 nested stacks. Change-Id: I2b00285514235834131222012408d2b5b2b37d30 Partial-Bug: 1489548
This commit is contained in:
parent
e84f7e4661
commit
1b2cd7495d
@ -0,0 +1,48 @@
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = sqlalchemy.MetaData(bind=migrate_engine)
|
||||
|
||||
res_table = sqlalchemy.Table('resource', meta, autoload=True)
|
||||
stack_table = sqlalchemy.Table('stack', meta, autoload=True)
|
||||
root_stack_id = sqlalchemy.Column('root_stack_id',
|
||||
sqlalchemy.String(36))
|
||||
|
||||
root_stack_id.create(res_table)
|
||||
root_stack_idx = sqlalchemy.Index('ix_resource_root_stack_id',
|
||||
res_table.c.root_stack_id,
|
||||
mysql_length=36)
|
||||
root_stack_idx.create(migrate_engine)
|
||||
|
||||
# build stack->owner relationship for all stacks
|
||||
stmt = sqlalchemy.select([stack_table.c.id, stack_table.c.owner_id])
|
||||
stacks = migrate_engine.execute(stmt)
|
||||
parent_stacks = dict([(s.id, s.owner_id) for s in stacks])
|
||||
|
||||
def root_for_stack(stack_id):
|
||||
owner_id = parent_stacks.get(stack_id)
|
||||
if owner_id:
|
||||
return root_for_stack(owner_id)
|
||||
return stack_id
|
||||
|
||||
# for each stack, update the resources with the root_stack_id
|
||||
for stack_id, owner_id in parent_stacks.items():
|
||||
root_id = root_for_stack(stack_id)
|
||||
values = {'root_stack_id': root_id}
|
||||
update = res_table.update().where(
|
||||
res_table.c.stack_id == stack_id).values(values)
|
||||
migrate_engine.execute(update)
|
@ -287,6 +287,7 @@ class Resource(BASE, HeatBase, StateAware):
|
||||
sqlalchemy.ForeignKey('stack.id'),
|
||||
nullable=False)
|
||||
stack = relationship(Stack, backref=backref('resources'))
|
||||
root_stack_id = sqlalchemy.Column(sqlalchemy.String(36), index=True)
|
||||
data = relationship(ResourceData,
|
||||
cascade="all,delete",
|
||||
backref=backref('resource'))
|
||||
|
@ -204,6 +204,7 @@ class Resource(object):
|
||||
self.replaces = None
|
||||
self.replaced_by = None
|
||||
self.current_template_id = None
|
||||
self.root_stack_id = None
|
||||
|
||||
if not stack.has_cache_data(name):
|
||||
resource = stack.db_resource_get(name)
|
||||
@ -243,6 +244,7 @@ class Resource(object):
|
||||
self.replaces = resource.replaces
|
||||
self.replaced_by = resource.replaced_by
|
||||
self.current_template_id = resource.current_template_id
|
||||
self.root_stack_id = resource.root_stack_id
|
||||
|
||||
@property
|
||||
def stack(self):
|
||||
@ -295,7 +297,8 @@ class Resource(object):
|
||||
'action': self.INIT,
|
||||
'status': self.COMPLETE,
|
||||
'current_template_id': new_tmpl_id,
|
||||
'stack_name': self.stack.name}
|
||||
'stack_name': self.stack.name,
|
||||
'root_stack_id': self.root_stack_id}
|
||||
new_rs = resource_objects.Resource.create(self.context, rs)
|
||||
|
||||
# 2. update the current resource to be replaced_by the one above.
|
||||
@ -1268,6 +1271,8 @@ class Resource(object):
|
||||
properties_data_encrypted, properties_data = \
|
||||
resource_objects.Resource.encrypt_properties_data(
|
||||
self._stored_properties_data)
|
||||
if not self.root_stack_id:
|
||||
self.root_stack_id = self.stack.root_stack_id()
|
||||
try:
|
||||
rs = {'action': self.action,
|
||||
'status': self.status,
|
||||
@ -1283,7 +1288,8 @@ class Resource(object):
|
||||
'replaces': self.replaces,
|
||||
'replaced_by': self.replaced_by,
|
||||
'current_template_id': self.current_template_id,
|
||||
'stack_name': self.stack.name}
|
||||
'stack_name': self.stack.name,
|
||||
'root_stack_id': self.root_stack_id}
|
||||
|
||||
new_rs = resource_objects.Resource.create(self.context, rs)
|
||||
self.id = new_rs.id
|
||||
@ -1323,7 +1329,8 @@ class Resource(object):
|
||||
'replaces': self.replaces,
|
||||
'replaced_by': self.replaced_by,
|
||||
'current_template_id': self.current_template_id,
|
||||
'nova_instance': self.resource_id
|
||||
'nova_instance': self.resource_id,
|
||||
'root_stack_id': self.root_stack_id
|
||||
}
|
||||
if prev_action == self.INIT:
|
||||
metadata = self.t.metadata()
|
||||
|
@ -248,9 +248,8 @@ class StackResource(resource.Resource):
|
||||
def _validate_nested_resources(self, templ):
|
||||
if cfg.CONF.max_resources_per_stack == -1:
|
||||
return
|
||||
root_stack_id = self.stack.root_stack_id()
|
||||
total_resources = (len(templ[templ.RESOURCES]) +
|
||||
self.stack.total_resources(root_stack_id))
|
||||
self.stack.total_resources(self.root_stack_id))
|
||||
|
||||
if self.nested():
|
||||
# It's an update and these resources will be deleted
|
||||
|
@ -64,6 +64,7 @@ class Resource(
|
||||
'requires': heat_fields.ListField(nullable=True, default=None),
|
||||
'replaces': fields.IntegerField(nullable=True),
|
||||
'replaced_by': fields.IntegerField(nullable=True),
|
||||
'root_stack_id': fields.StringField(nullable=True),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
@ -619,6 +619,60 @@ class HeatMigrationsCheckers(test_migrations.WalkVersionsMixin,
|
||||
self.assertColumnNotExists(engine, 'raw_template',
|
||||
'predecessor')
|
||||
|
||||
def _pre_upgrade_065(self, engine):
|
||||
raw_template = utils.get_table(engine, 'raw_template')
|
||||
templ = []
|
||||
for i in range(960, 963, 1):
|
||||
t = dict(id=i, template='{}', files='{}')
|
||||
engine.execute(raw_template.insert(), [t])
|
||||
templ.append(t)
|
||||
|
||||
user_creds = utils.get_table(engine, 'user_creds')
|
||||
user = [dict(id=uid, username='test_user', password='password',
|
||||
tenant='test_project', auth_url='bla',
|
||||
tenant_id=str(uuid.uuid4()),
|
||||
trust_id='',
|
||||
trustor_user_id='') for uid in range(960, 963)]
|
||||
engine.execute(user_creds.insert(), user)
|
||||
|
||||
stack = utils.get_table(engine, 'stack')
|
||||
root_sid = '9a6a3ddb-2219-452c-8fec-a4977f8fe474'
|
||||
stack_ids = [(root_sid, 0, None),
|
||||
('b6a23bc2-cd4e-496f-be2e-c11d06124ea2', 1, root_sid),
|
||||
('7a927947-e004-4afa-8d11-62c1e049ecbd', 2, root_sid)]
|
||||
data = [dict(id=ll_id, name=ll_id,
|
||||
owner_id=owner_id,
|
||||
raw_template_id=templ[templ_id]['id'],
|
||||
user_creds_id=user[templ_id]['id'],
|
||||
username='test_user',
|
||||
disable_rollback=True,
|
||||
parameters='test_params',
|
||||
created_at=datetime.datetime.utcnow(),
|
||||
deleted_at=None)
|
||||
for ll_id, templ_id, owner_id in stack_ids]
|
||||
|
||||
engine.execute(stack.insert(), data)
|
||||
|
||||
res_table = utils.get_table(engine, 'resource')
|
||||
resource_ids = [(960, root_sid),
|
||||
(961, 'b6a23bc2-cd4e-496f-be2e-c11d06124ea2'),
|
||||
(962, '7a927947-e004-4afa-8d11-62c1e049ecbd')]
|
||||
resources = [dict(id=rid, stack_id=sid)
|
||||
for rid, sid in resource_ids]
|
||||
engine.execute(res_table.insert(), resources)
|
||||
|
||||
def _check_065(self, engine, data):
|
||||
self.assertColumnExists(engine, 'resource', 'root_stack_id')
|
||||
res_table = utils.get_table(engine, 'resource')
|
||||
res_in_db = list(res_table.select().execute())
|
||||
self.assertTrue(len(res_in_db) >= 3)
|
||||
# confirm the resource.root_stack_id is set for all resources
|
||||
for r in res_in_db:
|
||||
self.assertTrue(r.root_stack_id is not None)
|
||||
if r.id >= 960 and r.id <= 962:
|
||||
root_stack_id = '9a6a3ddb-2219-452c-8fec-a4977f8fe474'
|
||||
self.assertEqual(root_stack_id, r.root_stack_id)
|
||||
|
||||
|
||||
class TestHeatMigrationsMySQL(HeatMigrationsCheckers,
|
||||
test_base.MySQLOpportunisticTestCase):
|
||||
|
@ -84,9 +84,8 @@ Outputs:
|
||||
stack.store()
|
||||
return stack
|
||||
|
||||
@mock.patch.object(parser.Stack, 'root_stack_id')
|
||||
@mock.patch.object(parser.Stack, 'total_resources')
|
||||
def test_nested_stack_three_deep(self, tr, rsi):
|
||||
def test_nested_stack_three_deep(self, tr):
|
||||
root_template = '''
|
||||
HeatTemplateFormatVersion: 2012-12-12
|
||||
Resources:
|
||||
@ -118,7 +117,6 @@ Resources:
|
||||
depth2_template,
|
||||
self.nested_template]
|
||||
|
||||
rsi.return_value = '1234'
|
||||
tr.return_value = 2
|
||||
|
||||
self.validate_stack(root_template)
|
||||
@ -126,11 +124,9 @@ Resources:
|
||||
mock.call('https://server.test/depth2.template'),
|
||||
mock.call('https://server.test/depth3.template')]
|
||||
urlfetch.get.assert_has_calls(calls)
|
||||
tr.assert_called_with('1234')
|
||||
|
||||
@mock.patch.object(parser.Stack, 'root_stack_id')
|
||||
@mock.patch.object(parser.Stack, 'total_resources')
|
||||
def test_nested_stack_six_deep(self, tr, rsi):
|
||||
def test_nested_stack_six_deep(self, tr):
|
||||
tmpl = '''
|
||||
HeatTemplateFormatVersion: 2012-12-12
|
||||
Resources:
|
||||
@ -158,11 +154,12 @@ Resources:
|
||||
depth5_template,
|
||||
self.nested_template]
|
||||
|
||||
rsi.return_value = '1234'
|
||||
tr.return_value = 5
|
||||
|
||||
t = template_format.parse(root_template)
|
||||
stack = self.parse_stack(t)
|
||||
stack['Nested'].root_stack_id = '1234'
|
||||
|
||||
res = self.assertRaises(exception.StackValidationFailed,
|
||||
stack.validate)
|
||||
self.assertIn('Recursion depth exceeds', six.text_type(res))
|
||||
@ -213,9 +210,8 @@ Resources:
|
||||
mock.call('https://server.test/depth4.template')]
|
||||
urlfetch.get.assert_has_calls(calls, any_order=True)
|
||||
|
||||
@mock.patch.object(parser.Stack, 'root_stack_id')
|
||||
@mock.patch.object(parser.Stack, 'total_resources')
|
||||
def test_nested_stack_infinite_recursion(self, tr, rsi):
|
||||
def test_nested_stack_infinite_recursion(self, tr):
|
||||
tmpl = '''
|
||||
HeatTemplateFormatVersion: 2012-12-12
|
||||
Resources:
|
||||
@ -227,7 +223,7 @@ Resources:
|
||||
urlfetch.get.return_value = tmpl
|
||||
t = template_format.parse(tmpl)
|
||||
stack = self.parse_stack(t)
|
||||
rsi.return_value = '1234'
|
||||
stack['Nested'].root_stack_id = '1234'
|
||||
tr.return_value = 2
|
||||
res = self.assertRaises(exception.StackValidationFailed,
|
||||
stack.validate)
|
||||
|
Loading…
x
Reference in New Issue
Block a user