openstack/magnum
Code Issues Proposed changes

Merge "Fix global stack list in periodic task"

Browse Source
This commit is contained in:
Jenkins 2016-07-27 11:47:50 +00:00 committed by Gerrit Code Review
commit 4b4d4b0f84
13 changed files with 191 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/source/install-guide-from-source.rst
View File

@ -466,17 +466,7 @@ Install and configure components
# su -s /bin/sh -c "/var/lib/magnum/env/bin/magnum-db-manage upgrade" magnum
9. Update heat policy to allow magnum list stacks. Edit your heat policy file,
usually ``/etc/heat/policy.json``:
.. code-block:: ini
...
stacks:global_index: "role:admin",
Now restart heat.
10. Set magnum for log rotation:
9. Set magnum for log rotation:
.. code-block:: console

28
doc/source/userguide.rst
View File

@ -1025,8 +1025,34 @@ High Availability
=======
Scaling
=======
*To be filled in*
Performance tuning for periodic task
------------------------------------
Magnum's periodic task performs a `stack-get` operation on the Heat stack
underlying each of its bays. If you have a large amount of bays this can create
considerable load on the Heat API. To reduce that load you can configure Magnum
to perform one global `stack-list` per periodic task instead instead of one per
bay. This is disabled by default, both from the Heat and Magnum side since it
causes a security issue, though: any user in any tenant holding the `admin`
role can perform a global `stack-list` operation if Heat is configured to allow
it for Magnum. If you want to enable it nonetheless, proceed as follows:
1. Set `periodic_global_stack_list` in magnum.conf to `True`
(`False` by default).
2. Update heat policy to allow magnum list stacks. To this end, edit your heat
policy file, usually etc/heat/policy.json``:
.. code-block:: ini
...
stacks:global_index: "role:admin",
Now restart heat.
*To be filled in*
Include auto scaling
=======

9
install-guide/source/common/configure_4_update_heat_policy.rst
View File

@ -1,9 +0,0 @@
4. Update heat policy to allow magnum list stacks. Edit your heat policy file,
usually ``/etc/heat/policy.json``:
.. code-block:: ini
...
stacks:global_index: "role:admin",
Now restart heat.

2
install-guide/source/install-debian-manual.rst
View File

@ -21,8 +21,6 @@ Install and configure components
.. include:: common/configure_3_populate_database.rst
.. include:: common/configure_4_update_heat_policy.rst
Finalize installation
---------------------

2
install-guide/source/install-obs.rst
View File

@ -22,8 +22,6 @@ Install and configure components
.. include:: common/configure_3_populate_database.rst
.. include:: common/configure_4_update_heat_policy.rst
Finalize installation
---------------------

2
install-guide/source/install-rdo.rst
View File

@ -31,8 +31,6 @@ Install and configure components
.. include:: common/configure_3_populate_database.rst
.. include:: common/configure_4_update_heat_policy.rst
Finalize installation
---------------------

2
install-guide/source/install-ubuntu.rst
View File

@ -21,8 +21,6 @@ Install and configure components
.. include:: common/configure_3_populate_database.rst
.. include:: common/configure_4_update_heat_policy.rst
Finalize installation
---------------------

31
magnum/common/context.py
View File

@ -11,22 +11,30 @@
# under the License.
from eventlet.green import threading
from oslo_config import cfg
from oslo_context import context
CONF = cfg.CONF
class RequestContext(context.RequestContext):
"""Extends security contexts from the OpenStack common library."""
def __init__(self, auth_token=None, auth_url=None, domain_id=None,
domain_name=None, user_name=None, user_id=None,
user_domain_name=None, user_domain_id=None,
project_name=None, project_id=None, roles=None,
is_admin=False, read_only=False, show_deleted=False,
request_id=None, trust_id=None, auth_token_info=None,
all_tenants=False, **kwargs):
all_tenants=False, password=None, **kwargs):
"""Stores several additional request parameters:
:param domain_id: The ID of the domain.
:param domain_name: The name of the domain.
:param user_domain_id: The ID of the domain to
authenticate a user against.
:param user_domain_name: The name of the domain to
authenticate a user against.
"""
super(RequestContext, self).__init__(auth_token=auth_token,
@ -43,11 +51,14 @@ class RequestContext(context.RequestContext):
self.project_id = project_id
self.domain_id = domain_id
self.domain_name = domain_name
self.user_domain_id = user_domain_id
self.user_domain_name = user_domain_name
self.roles = roles
self.auth_url = auth_url
self.auth_token_info = auth_token_info
self.trust_id = trust_id
self.all_tenants = all_tenants
self.password = password
def to_dict(self):
value = super(RequestContext, self).to_dict()
@ -55,6 +66,8 @@ class RequestContext(context.RequestContext):
'auth_url': self.auth_url,
'domain_id': self.domain_id,
'domain_name': self.domain_name,
'user_domain_id': self.user_domain_id,
'user_domain_name': self.user_domain_name,
'user_name': self.user_name,
'user_id': self.user_id,
'project_name': self.project_name,
@ -66,6 +79,7 @@ class RequestContext(context.RequestContext):
'request_id': self.request_id,
'trust_id': self.trust_id,
'auth_token_info': self.auth_token_info,
'password': self.password,
'all_tenants': self.all_tenants})
return value
@ -91,6 +105,21 @@ def make_admin_context(show_deleted=False, all_tenants=False):
return context
def make_bay_context(bay, show_deleted=False):
"""Create a user context based on a bay's stored Keystone trust.
:param bay: the bay supplying the Keystone trust to use
:param show_deleted: if True, will show deleted items when query db
"""
context = RequestContext(user_name=bay.trustee_username,
password=bay.trustee_password,
trust_id=bay.trust_id,
show_deleted=show_deleted,
user_domain_id=CONF.trust.trustee_domain_id,
user_domain_name=CONF.trust.trustee_domain_name)
return context
_CTX_STORE = threading.local()
_CTX_KEY = 'current_ctx'

14
magnum/common/keystone.py
View File

@ -113,7 +113,7 @@ class KeystoneClientV3(object):
return session
def _get_auth(self):
if self.context.is_admin or self.context.trust_id:
if self.context.is_admin:
try:
auth = ka_loading.load_auth_from_conf_options(CONF, CFG_GROUP)
except ka_exception.MissingRequiredOptions:
@ -125,6 +125,18 @@ class KeystoneClientV3(object):
elif self.context.auth_token:
auth = ka_v3.Token(auth_url=self.auth_url,
token=self.context.auth_token)
elif self.context.trust_id:
auth_info = {
'auth_url': self.auth_url,
'username': self.context.user_name,
'password': self.context.password,
'user_domain_id': self.context.user_domain_id,
'user_domain_name': self.context.user_domain_name,
'trust_id': self.context.trust_id
}
auth = ka_v3.Password(**auth_info)
else:
LOG.error(_LE('Keystone API connection failed: no password, '
'trust_id or token found.'))

7
magnum/common/rpc_service.py
View File

@ -42,6 +42,13 @@ TRANSPORT_ALIASES = {
}
periodic_opts = [
cfg.BoolOpt('periodic_global_stack_list',
default=False,
help="List Heat stacks globally when syncing bays. "
"Default is to do retrieve each bay's stack "
"individually. Reduces number of requests against "
"Heat API if enabled but requires changes to Heat's "
"policy.json."),
cfg.BoolOpt('periodic_enable',
default=True,
help='Enable periodic tasks.'),

43
magnum/service/periodic.py
View File

@ -15,6 +15,8 @@
import functools
from heatclient import exc as heat_exc
from oslo_config import cfg
from oslo_log import log
from oslo_service import periodic_task
import six
@ -31,6 +33,7 @@ from magnum import objects
from magnum.objects.fields import BayStatus as bay_status
CONF = cfg.CONF
LOG = log.getLogger(__name__)
@ -80,8 +83,14 @@ class MagnumPeriodicTasks(periodic_task.PeriodicTasks):
sid_to_bay_mapping = {bay.stack_id: bay for bay in bays}
bay_stack_ids = sid_to_bay_mapping.keys()
stacks = osc.heat().stacks.list(global_tenant=True,
filters={'id': bay_stack_ids})
if CONF.periodic_global_stack_list:
stacks = osc.heat().stacks.list(global_tenant=True,
filters={'id': bay_stack_ids})
else:
ret = self._get_bay_stacks(bays, sid_to_bay_mapping,
bay_stack_ids)
[stacks, bays, bay_stack_ids, sid_to_bay_mapping] = ret
sid_to_stack_mapping = {s.id: s for s in stacks}
# intersection of bays magnum has and heat has
@ -102,6 +111,36 @@ class MagnumPeriodicTasks(periodic_task.PeriodicTasks):
"Ignore error [%s] when syncing up bay status."
), e, exc_info=True)
def _get_bay_stacks(self, bays, sid_to_bay_mapping, bay_stack_ids):
stacks = []
_bays = bays
_sid_to_bay_mapping = sid_to_bay_mapping
_bay_stack_ids = bay_stack_ids
for bay in _bays:
try:
# Create client with bay's trustee user context
bosc = clients.OpenStackClients(
context.make_bay_context(bay))
stack = bosc.heat().stacks.get(bay.stack_id)
stacks.append(stack)
# No need to do anything in this case
except heat_exc.HTTPNotFound:
pass
except Exception as e:
# Any other exception means we do not perform any
# action on this bay in the current sync run, so remove
# it from all records.
LOG.warning("Exception while attempting to retrieve "
"Heat stack %s for bay %s. Traceback "
"follows.")
LOG.warning(e)
_sid_to_bay_mapping.pop(bay.stack_id)
_bay_stack_ids.remove(bay.stack_id)
_bays.remove(bay)
return [stacks, _bays, _bay_stack_ids, _sid_to_bay_mapping]
def _sync_existing_bay(self, bay, stack):
if bay.status != stack.stack_status:
old_status = bay.status

10
magnum/tests/unit/db/utils.py
View File

@ -73,7 +73,7 @@ def create_test_baymodel(**kw):
def get_test_bay(**kw):
return {
attrs = {
'id': kw.get('id', 42),
'uuid': kw.get('uuid', '5d12f6fd-a196-4bf0-ae4c-1f639a523a52'),
'name': kw.get('name', 'bay1'),
@ -97,6 +97,14 @@ def get_test_bay(**kw):
'updated_at': kw.get('updated_at'),
}
# Only add Keystone trusts related attributes on demand since they may
# break other tests.
for attr in ['trustee_username', 'trustee_password', 'trust_id']:
if attr in kw:
attrs[attr] = kw[attr]
return attrs
def create_test_bay(**kw):
"""Create test bay entry in DB and return Bay DB object.

71
magnum/tests/unit/service/test_periodic.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from heatclient import exc as heat_exc
import mock
from magnum.common import context
@ -37,14 +38,25 @@ class PeriodicTestCase(base.TestCase):
ctx = context.make_admin_context()
bay1 = utils.get_test_bay(id=1, stack_id='11',
status=bay_status.CREATE_IN_PROGRESS)
bay2 = utils.get_test_bay(id=2, stack_id='22',
status=bay_status.DELETE_IN_PROGRESS)
bay3 = utils.get_test_bay(id=3, stack_id='33',
status=bay_status.UPDATE_IN_PROGRESS)
bay4 = utils.get_test_bay(id=4, stack_id='44',
status=bay_status.CREATE_COMPLETE)
# Can be identical for all bays.
trust_attrs = {
'trustee_username': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52',
'trustee_password': 'ain7einaebooVaig6d',
'trust_id': '39d920ca-67c6-4047-b57a-01e9e16bb96f',
}
trust_attrs.update({'id': 1, 'stack_id': '11',
'status': bay_status.CREATE_IN_PROGRESS})
bay1 = utils.get_test_bay(**trust_attrs)
trust_attrs.update({'id': 2, 'stack_id': '22',
'status': bay_status.DELETE_IN_PROGRESS})
bay2 = utils.get_test_bay(**trust_attrs)
trust_attrs.update({'id': 3, 'stack_id': '33',
'status': bay_status.UPDATE_IN_PROGRESS})
bay3 = utils.get_test_bay(**trust_attrs)
trust_attrs.update({'id': 4, 'stack_id': '44',
'status': bay_status.CREATE_COMPLETE})
bay4 = utils.get_test_bay(**trust_attrs)
self.bay1 = objects.Bay(ctx, **bay1)
self.bay2 = objects.Bay(ctx, **bay2)
@ -63,6 +75,14 @@ class PeriodicTestCase(base.TestCase):
stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE,
stack_status_reason='fake_reason_33')
mock_heat_client.stacks.list.return_value = [stack1, stack3]
get_stacks = {'11': stack1, '33': stack3}
def stack_get_sideefect(arg):
if arg == '22':
raise heat_exc.HTTPNotFound
return get_stacks[arg]
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
@ -79,6 +99,33 @@ class PeriodicTestCase(base.TestCase):
self.assertEqual(bay_status.UPDATE_COMPLETE, self.bay3.status)
self.assertEqual('fake_reason_33', self.bay3.status_reason)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')
def test_sync_auth_fail(self, mock_oscc, mock_bay_list):
"""Tests handling for unexpected exceptions in _get_bay_stacks()
It does this by raising an a HTTPUnauthorized exception in Heat client.
The affected stack thus missing from the stack list should not lead to
bay state changing in this case. Likewise, subsequent bays should still
change state, despite the affected bay being skipped.
"""
stack1 = fake_stack(id='11',
stack_status=bay_status.CREATE_COMPLETE)
mock_heat_client = mock.MagicMock()
def stack_get_sideefect(arg):
raise heat_exc.HTTPUnauthorized
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_heat_client.stacks.list.return_value = [stack1]
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1]
periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
self.assertEqual(bay_status.CREATE_IN_PROGRESS, self.bay1.status)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')
def test_sync_bay_status_not_changes(self, mock_oscc, mock_bay_list):
@ -89,6 +136,14 @@ class PeriodicTestCase(base.TestCase):
stack_status=bay_status.DELETE_IN_PROGRESS)
stack3 = fake_stack(id='33',
stack_status=bay_status.UPDATE_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3}
def stack_get_sideefect(arg):
if arg == '22':
raise heat_exc.HTTPNotFound
return get_stacks[arg]
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3]
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client