openstack/heat
Code Issues Proposed changes

Revert "Implement stack-locking for multi-engine support"

Browse Source 
This is a short term backout until it gets fixed.

This reverts commit c30530f126.
Closes-bug: #1250797

Change-Id: Ide1dc4a9d1469dee033e8bc08d8ab4da96f79fc0
This commit is contained in:
Angus Salkeld 2013-11-13 22:59:34 +11:00
parent c30530f126
commit 4ed487bef4
12 changed files with 0 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
etc/heat/heat.conf.sample
View File

@ -10,11 +10,6 @@
# Driver to use for controlling instances (string value)
#instance_driver=heat.engine.nova
# Engine identifier for multi-engine distributed lock. If
# this is set to "generate_uuid", a UUID will be generated.
# (string value)
#engine_id=generate_uuid
# List of directories to search for Plugins (list value)
#plugin_dirs=/usr/lib64/heat,/usr/lib/heat

13
heat/api/aws/exception.py
View File

@ -247,16 +247,6 @@ class HeatAPINotImplementedError(HeatAPIException):
err_type = "Server"
class HeatActionInProgressError(HeatAPIException):
'''
Cannot perform action on stack in its current state
'''
code = 400
title = 'InvalidAction'
explanation = ("Cannot perform action on stack while other actions are " +
"in progress")
def map_remote_error(ex):
"""
Map rpc_common.RemoteError exceptions returned by the engine
@ -281,7 +271,6 @@ def map_remote_error(ex):
)
denied_errors = ('Forbidden', 'NotAuthorized')
already_exists_errors = ('StackExists')
invalid_action_errors = ('ActionInProgress',)
ex_type = ex.__class__.__name__
@ -294,8 +283,6 @@ def map_remote_error(ex):
return HeatAccessDeniedError(detail=str(ex))
elif ex_type in already_exists_errors:
return AlreadyExistsError(detail=str(ex))
elif ex_type in invalid_action_errors:
return HeatActionInProgressError(detail=str(ex))
else:
# Map everything else to internal server error for now
return HeatInternalFailureError(detail=str(ex))

1
heat/api/middleware/fault.py
View File

@ -58,7 +58,6 @@ class FaultWrapper(wsgi.Middleware):
error_map = {
'AttributeError': webob.exc.HTTPBadRequest,
'ActionInProgress': webob.exc.HTTPConflict,
'ValueError': webob.exc.HTTPBadRequest,
'StackNotFound': webob.exc.HTTPNotFound,
'ResourceNotFound': webob.exc.HTTPNotFound,

5
heat/common/config.py
View File

@ -75,11 +75,6 @@ engine_opts = [
cfg.StrOpt('instance_driver',
default='heat.engine.nova',
help='Driver to use for controlling instances'),
cfg.StrOpt('engine_id',
default="generate_uuid",
help=_('Engine identifier for multi-engine distributed lock.'
' If this is set to "generate_uuid", a UUID will be'
' generated.')),
cfg.ListOpt('plugin_dirs',
default=['/usr/lib64/heat', '/usr/lib/heat'],
help='List of directories to search for Plugins'),

5
heat/common/exception.py
View File

@ -334,8 +334,3 @@ class RequestLimitExceeded(HeatException):
class StackResourceLimitExceeded(HeatException):
msg_fmt = _('Maximum resources per stack exceeded.')
class ActionInProgress(HeatException):
msg_fmt = _("Stack %(stack_name)s already has an action (%(action)s) "
"in progress.")

20
heat/db/api.py
View File

@ -139,22 +139,6 @@ def stack_delete(context, stack_id):
return IMPL.stack_delete(context, stack_id)
def stack_lock_get(context, stack_id):
return IMPL.stack_lock_get(context, stack_id)
def stack_lock_create(context, stack_id, engine_id):
return IMPL.stack_lock_create(context, stack_id, engine_id)
def stack_lock_steal(context, stack_id, engine_id):
return IMPL.stack_lock_steal(context, stack_id, engine_id)
def stack_lock_release(context, stack_id):
return IMPL.stack_lock_release(context, stack_id)
def user_creds_create(context):
return IMPL.user_creds_create(context)
@ -231,7 +215,3 @@ def db_sync(version=None):
def db_version():
"""Display the current database version."""
return IMPL.db_version()
def current_timestamp():
return IMPL.current_timestamp()

30
heat/db/sqlalchemy/api.py
View File

@ -297,28 +297,6 @@ def stack_delete(context, stack_id):
session.flush()
def stack_lock_get(context, stack_id):
return model_query(context, models.StackLock).get(stack_id)
def stack_lock_create(context, stack_id, engine_id):
stack_lock = models.StackLock()
stack_lock.update({"stack_id": stack_id,
"engine_id": engine_id})
stack_lock.save(_session(context))
def stack_lock_steal(context, stack_id, engine_id):
stack_lock = stack_lock_get(context, stack_id)
stack_lock.update({"engine_id": engine_id})
stack_lock.save(_session(context))
def stack_lock_release(context, stack_id):
stack_lock = stack_lock_get(context, stack_id)
stack_lock.delete()
def user_creds_create(context):
values = context.to_dict()
user_creds_ref = models.UserCreds()
@ -543,11 +521,3 @@ def db_sync(version=None):
def db_version():
"""Display the current database version."""
return migration.db_version()
def current_timestamp():
"""Return a datetime object with the current database time."""
session = get_session()
query = sqlalchemy.select([sqlalchemy.func.current_timestamp()])
result = session.execute(query).fetchall()
return result[0][0]

40
heat/db/sqlalchemy/migrate_repo/versions/029_stack_lock.py
View File

@ -1,40 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy
def upgrade(migrate_engine):
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
stack_lock = sqlalchemy.Table(
'stack_lock', meta,
sqlalchemy.Column('stack_id', sqlalchemy.String(length=36),
sqlalchemy.ForeignKey('stack.id'),
primary_key=True,
nullable=False),
sqlalchemy.Column('created_at', sqlalchemy.DateTime),
sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
sqlalchemy.Column('engine_id', sqlalchemy.String(length=64))
)
sqlalchemy.Table('stack', meta, autoload=True)
stack_lock.create()
def downgrade(migrate_engine):
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
stack_lock = sqlalchemy.Table('stack_lock', meta, autoload=True)
stack_lock.drop()

11
heat/db/sqlalchemy/models.py
View File

@ -143,17 +143,6 @@ class Stack(BASE, HeatBase, SoftDelete):
disable_rollback = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False)
class StackLock(BASE, HeatBase):
"""Store stack locks for deployments with multiple-engines."""
__tablename__ = 'stack_lock'
stack_id = sqlalchemy.Column(sqlalchemy.String,
sqlalchemy.ForeignKey('stack.id'),
primary_key=True)
engine_id = sqlalchemy.Column(sqlalchemy.String)
class UserCreds(BASE, HeatBase):
"""
Represents user credentials and mirrors the 'context'

89
heat/engine/stack_lock.py
View File

@ -1,89 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
cfg.CONF.import_opt('engine_id', 'heat.common.config')
from heat.common import exception
from heat.db import api as db_api
from heat.openstack.common import log as logging
from heat.openstack.common import uuidutils
logger = logging.getLogger(__name__)
class StackLock(object):
def __init__(self, context, stack):
self.context = context
self.stack = stack
if cfg.CONF.engine_id == "generate_uuid":
self.engine_id = uuidutils.generate_uuid()
else:
self.engine_id = cfg.CONF.engine_id
@staticmethod
def _lock_staleness(lock):
"""Returns number of seconds since stack was created or updated."""
if lock.updated_at:
changed_time = lock.updated_at
else:
changed_time = lock.created_at
current_time = db_api.current_timestamp()
current_epoch = float(current_time.strftime('%s'))
changed_epoch = float(changed_time.strftime('%s'))
return current_epoch - changed_epoch
@property
def timeout(self):
"""Returns the stack timeout in seconds."""
try:
return self.stack.timeout * 60
except AttributeError:
return self.stack.timeout_secs()
def acquire(self):
"""Acquire a lock on the stack."""
existing_lock = db_api.stack_lock_get(self.context, self.stack.id)
if existing_lock:
if self._lock_staleness(existing_lock) > self.timeout:
logger.info("Lock expired. Engine %s is stealing the lock"
% existing_lock.engine_id)
db_api.stack_lock_steal(self.context, self.stack.id,
self.engine_id)
else:
logger.debug("Stack lock is owned by engine %s"
% existing_lock.engine_id)
raise exception.ActionInProgress(stack_name=self.stack.name,
action=self.stack.action)
else:
db_api.stack_lock_create(self.context, self.stack.id,
self.engine_id)
logger.debug("Acquired lock for engine: %s, stack: %s, action: %s"
% (self.engine_id, self.stack.id, self.stack.action))
def release(self):
"""Release a stack lock."""
logger.debug("Releasing lock for engine: %s, stack: %s, action: %s"
% (self.engine_id, self.stack.id, self.stack.action))
db_api.stack_lock_release(self.context, self.stack.id)
def _gt_callback_release(self, gt, *args, **kwargs):
"""Callback function that will be passed to GreenThread.link()."""
# If gt.wait() isn't called here and a lock exists, then the
# pending _gt_callback_release() from the previous acquire()
# will be executed immediately upon the next call to
# acquire(). This leads to a pre-mature release of the lock.
gt.wait()
self.release()

45
heat/tests/test_sqlalchemy_api.py
View File

@ -17,7 +17,6 @@ import fixtures
from json import loads
from json import dumps
import mox
from testtools import matchers
from heat.db.sqlalchemy import api as db_api
@ -419,10 +418,6 @@ def create_resource_data(ctx, resource, **kwargs):
return db_api.resource_data_set(resource, **values)
def create_stack_lock(ctx, stack_id, engine_id):
return db_api.stack_lock_create(ctx, stack_id, engine_id)
def create_event(ctx, **kwargs):
values = {
'stack_id': 'test_stack_id',
@ -805,34 +800,6 @@ class DBAPIResourceTest(HeatTestCase):
self.ctx, self.stack2.id)
class DBAPIStackLockTest(HeatTestCase):
def setUp(self):
super(DBAPIStackLockTest, self).setUp()
self.ctx = utils.dummy_context()
utils.setup_dummy_db()
utils.reset_dummy_db()
self.template = create_raw_template(self.ctx)
self.user_creds = create_user_creds(self.ctx)
self.stack = create_stack(self.ctx, self.template, self.user_creds)
def test_stack_lock_create_get(self):
create_stack_lock(self.ctx, self.stack.id, UUID1)
lock = db_api.stack_lock_get(self.ctx, self.stack.id)
self.assertEqual(UUID1, lock['engine_id'])
def test_stack_lock_steal(self):
create_stack_lock(self.ctx, self.stack.id, UUID1)
db_api.stack_lock_steal(self.ctx, self.stack.id, UUID2)
lock = db_api.stack_lock_get(self.ctx, self.stack.id)
self.assertEqual(UUID2, lock['engine_id'])
def test_stack_lock_release(self):
create_stack_lock(self.ctx, self.stack.id, UUID1)
db_api.stack_lock_release(self.ctx, self.stack.id)
lock = db_api.stack_lock_get(self.ctx, self.stack.id)
self.assertIsNone(lock)
class DBAPIResourceDataTest(HeatTestCase):
def setUp(self):
super(DBAPIResourceDataTest, self).setUp()
@ -1083,15 +1050,3 @@ class DBAPIWatchDataTest(HeatTestCase):
data = [wd.data for wd in watch_data]
[self.assertIn(val['data'], data) for val in values]
class DBAPIUtilTest(HeatTestCase):
def setUp(self):
super(DBAPIUtilTest, self).setUp()
self.ctx = utils.dummy_context()
utils.setup_dummy_db()
utils.reset_dummy_db()
def test_current_timestamp(self):
current_timestamp = db_api.current_timestamp()
self.assertThat(current_timestamp, matchers.IsInstance(datetime))

93
heat/tests/test_stack_lock.py
View File

@ -1,93 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import mox
from heat.common import exception
from heat.db import api as db_api
from heat.engine import stack_lock
from heat.tests.common import HeatTestCase
from heat.tests import utils
class StackLockTest(HeatTestCase):
def setUp(self):
super(StackLockTest, self).setUp()
utils.setup_dummy_db()
self.context = utils.dummy_context()
self.stack = self.m.CreateMockAnything()
self.stack.id = "aae01f2d-52ae-47ac-8a0d-3fde3d220fea"
self.stack.name = "test_stack"
self.stack.action = "CREATE"
self.stack.timeout = 1
def test_successful_acquire_new_lock(self):
self.m.StubOutWithMock(db_api, "stack_lock_get")
db_api.stack_lock_get(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(None)
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack)
slock.acquire()
self.m.VerifyAll()
def test_successful_acquire_steal_lock_updated(self):
existing_lock = self.m.CreateMockAnything()
existing_lock.updated_at = datetime.datetime(2012, 10, 16, 18, 35, 18)
current_time = datetime.datetime(2012, 10, 16, 18, 36, 29)
self.m.StubOutWithMock(db_api, "current_timestamp")
db_api.current_timestamp().AndReturn(current_time)
self.m.StubOutWithMock(db_api, "stack_lock_get")
db_api.stack_lock_get(mox.IgnoreArg(), mox.IgnoreArg())\
.AndReturn(existing_lock)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack)
slock.acquire()
self.m.VerifyAll()
def test_successful_acquire_steal_lock_created(self):
existing_lock = self.m.CreateMockAnything()
existing_lock.updated_at = None
existing_lock.created_at = datetime.datetime(2012, 10, 16, 18, 35, 18)
current_time = datetime.datetime(2012, 10, 16, 18, 36, 29)
self.m.StubOutWithMock(db_api, "current_timestamp")
db_api.current_timestamp().AndReturn(current_time)
self.m.StubOutWithMock(db_api, "stack_lock_get")
db_api.stack_lock_get(mox.IgnoreArg(), mox.IgnoreArg())\
.AndReturn(existing_lock)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack)
slock.acquire()
self.m.VerifyAll()
def test_failed_acquire(self):
existing_lock = self.m.CreateMockAnything()
existing_lock.updated_at = datetime.datetime(2012, 10, 16, 18, 35, 18)
current_time = datetime.datetime(2012, 10, 16, 18, 35, 29)
self.m.StubOutWithMock(db_api, "current_timestamp")
db_api.current_timestamp().AndReturn(current_time)
self.m.StubOutWithMock(db_api, "stack_lock_get")
db_api.stack_lock_get(mox.IgnoreArg(), mox.IgnoreArg())\
.AndReturn(existing_lock)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()