openstack/heat
Code Issues Proposed changes

Use root_stack_id for stack_count_total_resources

Browse Source 
The current implementation of stack_count_total_resources
scales very poorly for stacks with many nested stacks.

This change replaces the implementation with a single count
query filtering on the resource.root_stack_id column.

The following template was used for performance testing:

heat_template_version: 2015-04-30
description: >
  Stress test, create many stacks in a RG
parameters:
  count:
    type: number
    default: 4
resources:
  dummies:
    type: OS::Heat::ResourceGroup
    properties:
      count: {get_param: count}
      resource_def:
        type: dummy_node.yaml

 # dummy_node.yaml
heat_template_version: 2015-04-30
parameters:
resources:
  random:
    type: OS::Heat::RandomString
  randoms:
    type: OS::Heat::ResourceGroup
    properties:
      count: 1
      resource_def:
        type: OS::Heat::RandomString

This template was used to time the stack creation with
count=40 and count=80 against heat-master, the previous
change I2b00285514235834131222012408d2b5b2b37d30
and this change. Here are the results:

 Stack heat-master-40 CREATE_COMPLETE
real    1m9.103s

 Stack root-stack-id-40 CREATE_COMPLETE
real    0m59.233s

 Stack count-total-resources-40 CREATE_COMPLETE
real    0m43.308s

 Stack heat-master-80 CREATE_COMPLETE
real    2m47.190s

 Stack root-stack-id-80 CREATE_COMPLETE
real    2m16.743s

 Stack count-total-resources-80 CREATE_COMPLETE
real    1m15.288s

Also, the test template in bug #1489548 took 3 minutes
to create (vs the originally reported 13 minutes).

Change-Id: Iab3eaaba3ece16e14db3231f1c725bca3c8985c2
Closes-Bug: 1489548
This commit is contained in:
Steve Baker 2015-09-22 10:50:23 +12:00
parent 1b2cd7495d
commit 5576ffd249
2 changed files with 16 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
heat/db/sqlalchemy/api.py
View File

@ -585,25 +585,10 @@ def stack_get_root_id(context, stack_id):
def stack_count_total_resources(context, stack_id):
# start with a stack_get to confirm the context can access the stack
if stack_id is None or stack_get(context, stack_id) is None:
return 0
def nested_stack_ids(sid):
yield sid
for child in stack_get_all_by_owner_id(context, sid):
for stack in nested_stack_ids(child.id):
yield stack
stack_ids = list(nested_stack_ids(stack_id))
# count all resources which belong to the stacks
# count all resources which belong to the root stack
results = model_query(
context, models.Resource
).filter(
models.Resource.stack_id.in_(stack_ids)
).count()
).filter(models.Resource.root_stack_id == stack_id).count()
return results

43
heat/tests/db/test_sqlalchemy_api.py
View File

@ -1871,10 +1871,14 @@ class DBAPIStackTest(common.HeatTestCase):
def test_stack_count_total_resources(self):
def add_resources(stack, count):
def add_resources(stack, count, root_stack_id):
for i in range(count):
create_resource(
self.ctx, stack, name='%s-%s' % (stack.name, i))
self.ctx,
stack,
name='%s-%s' % (stack.name, i),
root_stack_id=root_stack_id
)
root = create_stack(self.ctx, self.template, self.user_creds,
name='root stack')
@ -1904,38 +1908,19 @@ class DBAPIStackTest(common.HeatTestCase):
s_4 = create_stack(self.ctx, self.template, self.user_creds,
name='s_4', owner_id=root.id)
add_resources(root, 3)
add_resources(s_1, 2)
add_resources(s_1_1, 4)
add_resources(s_1_2, 5)
add_resources(s_1_3, 6)
add_resources(root, 3, root.id)
add_resources(s_1, 2, root.id)
add_resources(s_1_1, 4, root.id)
add_resources(s_1_2, 5, root.id)
add_resources(s_1_3, 6, root.id)
add_resources(s_2, 1)
add_resources(s_2_1_1_1, 1)
add_resources(s_3, 4)
add_resources(s_2, 1, root.id)
add_resources(s_2_1_1_1, 1, root.id)
add_resources(s_3, 4, root.id)
self.assertEqual(26, db_api.stack_count_total_resources(
self.ctx, root.id))
self.assertEqual(17, db_api.stack_count_total_resources(
self.ctx, s_1.id))
self.assertEqual(4, db_api.stack_count_total_resources(
self.ctx, s_1_1.id))
self.assertEqual(5, db_api.stack_count_total_resources(
self.ctx, s_1_2.id))
self.assertEqual(6, db_api.stack_count_total_resources(
self.ctx, s_1_3.id))
self.assertEqual(2, db_api.stack_count_total_resources(
self.ctx, s_2.id))
self.assertEqual(1, db_api.stack_count_total_resources(
self.ctx, s_2_1.id))
self.assertEqual(1, db_api.stack_count_total_resources(
self.ctx, s_2_1_1.id))
self.assertEqual(1, db_api.stack_count_total_resources(
self.ctx, s_2_1_1_1.id))
self.assertEqual(4, db_api.stack_count_total_resources(
self.ctx, s_3.id))
self.assertEqual(0, db_api.stack_count_total_resources(
self.ctx, s_4.id))
self.assertEqual(0, db_api.stack_count_total_resources(