openstack/heat
Code Issues Proposed changes

Work around problems storing huge properties in events

Browse Source 
There is a maximum limit on the size of the resource properties we can
store for an event in MySQL.

To work around this, store an error instead of the largest property,
as this is expected to cater for graceful-failure of the most common
known case (large SoftwareConfig config properties), with a fallback
of storing only the error when this also fails.

Co-Authored-By: Steven Hardy <shardy@redhat.com>
Co-Authored-By: Marios Andreou <marios@redhat.com>
Closes-Bug: #1493858
Change-Id: I668c7ed8ca6c063fd20bc5271d6afea941a5f277
This commit is contained in:
Zane Bitter 2015-09-10 10:58:52 -04:00 committed by Steven Hardy
parent e7b89a485f
commit ab78bde75b
4 changed files with 115 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
heat/engine/event.py
View File

@ -13,6 +13,8 @@
import six
import oslo_db.exception
from heat.common import exception
from heat.common.i18n import _
from heat.common import identifier
@ -84,7 +86,22 @@ class Event(object):
if self.timestamp is not None:
ev['created_at'] = self.timestamp
new_ev = event_object.Event.create(self.context, ev)
try:
new_ev = event_object.Event.create(self.context, ev)
except oslo_db.exception.DBError:
# Attempt do drop the largest key and re-store as we expect
# This to mostly happen with one large config blob property
max_key, max_val = max(ev['resource_properties'].items(),
key=lambda i: len(repr(i[1])))
err = 'Resource properties are too large to store'
ev['resource_properties'].update({'Error': err})
ev['resource_properties'][max_key] = '<Deleted, too large>'
try:
new_ev = event_object.Event.create(self.context, ev)
except oslo_db.exception.DBError:
# Give up and drop all properties..
ev['resource_properties'] = {'Error': err}
new_ev = event_object.Event.create(self.context, ev)
self.id = new_ev.id
return self.id

3
heat/tests/common.py
View File

@ -151,6 +151,9 @@ class HeatTestCase(testscenarios.WithScenarios,
generic_rsrc.ResourceWithAttributeType)
resource._register_class('ResourceWithRequiredProps',
generic_rsrc.ResourceWithRequiredProps)
resource._register_class(
'ResourceWithMultipleRequiredProps',
generic_rsrc.ResourceWithMultipleRequiredProps)
resource._register_class(
'ResourceWithRequiredPropsAndEmptyAttrs',
generic_rsrc.ResourceWithRequiredPropsAndEmptyAttrs)

9
heat/tests/generic_resource.py
View File

@ -145,6 +145,15 @@ class ResourceWithRequiredProps(GenericResource):
required=True)}
class ResourceWithMultipleRequiredProps(GenericResource):
properties_schema = {'Foo1': properties.Schema(properties.Schema.STRING,
required=True),
'Foo2': properties.Schema(properties.Schema.STRING,
required=True),
'Foo3': properties.Schema(properties.Schema.STRING,
required=True)}
class ResourceWithRequiredPropsAndEmptyAttrs(GenericResource):
properties_schema = {'Foo': properties.Schema(properties.Schema.STRING,
required=True)}

88
heat/tests/test_event.py
View File

@ -14,6 +14,7 @@
import datetime
import mock
from oslo_config import cfg
import oslo_db.exception
from heat.common import exception
from heat.engine import event
@ -39,11 +40,25 @@ tmpl = {
}
}
tmpl_multiple = {
'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {
'EventTestResource': {
'Type': 'ResourceWithMultipleRequiredProps',
'Properties': {'Foo1': 'zoo',
'Foo2': 'A0000000000',
'Foo3': '99999'}
}
}
}
class EventTest(common.HeatTestCase):
class EventCommon(common.HeatTestCase):
def setUp(self):
super(EventTest, self).setUp()
super(EventCommon, self).setUp()
def _setup_stack(self, the_tmpl):
self.username = 'event_test_user'
self.ctx = utils.dummy_context()
@ -51,13 +66,20 @@ class EventTest(common.HeatTestCase):
self.m.ReplayAll()
self.stack = stack.Stack(self.ctx, 'event_load_test_stack',
template.Template(tmpl))
template.Template(the_tmpl))
self.stack.store()
self.resource = self.stack['EventTestResource']
self.resource._store()
self.addCleanup(stack_object.Stack.delete, self.ctx, self.stack.id)
class EventTest(EventCommon):
def setUp(self):
super(EventTest, self).setUp()
self._setup_stack(tmpl)
def test_load(self):
self.resource.resource_id_set('resource_physical_id')
@ -179,3 +201,63 @@ class EventTest(common.HeatTestCase):
e = event.Event(self.ctx, self.stack, 'TEST', 'IN_PROGRESS', 'Testing',
'wibble', res.properties, res.name, res.type())
self.assertIn('Error', e.resource_properties)
class EventTestProps(EventCommon):
def setUp(self):
super(EventTestProps, self).setUp()
self._setup_stack(tmpl_multiple)
def test_store_fail_all_props(self):
self.resource.resource_id_set('resource_physical_id')
e = event.Event(self.ctx, self.stack, 'TEST', 'IN_PROGRESS', 'Testing',
'alabama', self.resource.properties,
self.resource.name, self.resource.type())
e.store()
self.assertIsNotNone(e.id)
ev = event_object.Event.get_by_id(self.ctx, e.id)
errors = [oslo_db.exception.DBError, oslo_db.exception.DBError]
def side_effect(*args):
try:
raise errors.pop()
except IndexError:
self.assertEqual(
{'Error': 'Resource properties are too large to store'},
args[1]['resource_properties'])
return ev
with mock.patch("heat.objects.event.Event") as mock_event:
mock_event.create.side_effect = side_effect
e.store()
def test_store_fail_one_prop(self):
self.resource.resource_id_set('resource_physical_id')
e = event.Event(self.ctx, self.stack, 'TEST', 'IN_PROGRESS', 'Testing',
'alabama', self.resource.properties,
self.resource.name, self.resource.type())
e.store()
self.assertIsNotNone(e.id)
ev = event_object.Event.get_by_id(self.ctx, e.id)
errors = [oslo_db.exception.DBError]
def side_effect(*args):
try:
raise errors.pop()
except IndexError:
self.assertEqual(
{'Foo1': 'zoo',
'Foo2': '<Deleted, too large>',
'Foo3': '99999',
'Error': 'Resource properties are too large to store'},
args[1]['resource_properties'])
return ev
with mock.patch("heat.objects.event.Event") as mock_event:
mock_event.create.side_effect = side_effect
e.store()