openstack
/
zun
Code Issues Proposed changes

Record action finish 

In before, there is no way to detect if an action is finished or not.
This commit add tracking for each action and record on DB once
an action is finish.

Change-Id: I9945087cf8d1484c09e60d11b1ba0c84c1799f00
This commit is contained in:
Hongbin Lu 2019-04-13 23:09:20 +00:00
parent f473532b8f
commit 8ce2a4f3a9
7 changed files with 136 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
zun/common/utils.py
View File

@ -599,6 +599,25 @@ class EventReporter(object):
return False
class FinishAction(object):
"""Context manager to finish container actions."""
def __init__(self, context, action_name, *container_uuids):
self.context = context
self.action_name = action_name
self.container_uuids = container_uuids
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
for uuid in self.container_uuids:
objects.ContainerAction.action_finish(
self.context, uuid, self.action_name, exc_val=exc_val,
exc_tb=exc_tb, want_result=False)
return False
def get_wrapped_function(function):
"""Get the method at the bottom of a stack of decorators."""
if not hasattr(function, '__closure__') or not function.__closure__:
@ -622,7 +641,7 @@ def get_wrapped_function(function):
return _get_wrapped_function(function)
def wrap_container_event(prefix):
def wrap_container_event(prefix, finish_action=None):
"""Warps a method to log the event taken on the container, and result.
This decorator wraps a method to log the start and result of an event, as
@ -636,10 +655,18 @@ def wrap_container_event(prefix):
keyed_args = inspect.getcallargs(wrapped_func, self, context,
*args, **kwargs)
container_uuid = keyed_args['container'].uuid
event_name = '{0}_{1}'.format(prefix, function.__name__)
with EventReporter(context, event_name, container_uuid):
return function(self, context, *args, **kwargs)
if finish_action is not None:
with FinishAction(
context, finish_action, container_uuid
), EventReporter(
context, event_name, container_uuid
):
return function(self, context, *args, **kwargs)
else:
with EventReporter(context, event_name, container_uuid):
return function(self, context, *args, **kwargs)
return decorated_function
return helper

62
zun/compute/manager.py
View File

@ -15,8 +15,6 @@
import contextlib
import itertools
import math
import six
import time
from oslo_log import log as logging
@ -24,6 +22,7 @@ from oslo_service import periodic_task
from oslo_utils import excutils
from oslo_utils import timeutils
from oslo_utils import uuidutils
import six
from zun.common import consts
from zun.common import context
@ -34,6 +33,7 @@ from zun.common.utils import translate_exception
from zun.common.utils import wrap_container_event
from zun.common.utils import wrap_exception
from zun.compute import compute_node_tracker
from zun.compute import container_actions
import zun.conf
from zun.container import driver
from zun.image.glance import driver as glance
@ -250,15 +250,17 @@ class Manager(periodic_task.PeriodicTasks):
requested_volumes, container, run, pci_requests=None):
@utils.synchronized(container.uuid)
def do_container_create():
self._wait_for_volumes_available(context, requested_volumes,
container)
self._attach_volumes(context, container, requested_volumes)
self._check_support_disk_quota(context, container)
created_container = self._do_container_create(
context, container, requested_networks, requested_volumes,
pci_requests, limits)
if run:
self._do_container_start(context, created_container)
with utils.FinishAction(context, container_actions.CREATE,
container.uuid):
self._wait_for_volumes_available(context, requested_volumes,
container)
self._attach_volumes(context, container, requested_volumes)
self._check_support_disk_quota(context, container)
created_container = self._do_container_create(
context, container, requested_networks, requested_volumes,
pci_requests, limits)
if run:
self._do_container_start(context, created_container)
utils.spawn_n(do_container_create)
@ -532,7 +534,8 @@ class Manager(periodic_task.PeriodicTasks):
utils.spawn_n(do_add_security_group)
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.ADD_SECURITY_GROUP)
def _add_security_group(self, context, container, security_group):
LOG.debug('Adding security_group to container: %s', container.uuid)
with self._update_task_state(context, container, consts.SG_ADDING):
@ -548,7 +551,9 @@ class Manager(periodic_task.PeriodicTasks):
utils.spawn_n(do_remove_security_group)
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(
prefix='compute',
finish_action=container_actions.REMOVE_SECURITY_GROUP)
def _remove_security_group(self, context, container, security_group):
LOG.debug('Removing security_group from container: %s', container.uuid)
with self._update_task_state(context, container, consts.SG_REMOVING):
@ -575,7 +580,8 @@ class Manager(periodic_task.PeriodicTasks):
raise
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.REBOOT)
def _do_container_reboot(self, context, container, timeout):
LOG.debug('Rebooting container: %s', container.uuid)
with self._update_task_state(context, container,
@ -591,7 +597,8 @@ class Manager(periodic_task.PeriodicTasks):
utils.spawn_n(do_container_reboot)
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.STOP)
def _do_container_stop(self, context, container, timeout):
LOG.debug('Stopping container: %s', container.uuid)
with self._update_task_state(context, container,
@ -618,7 +625,8 @@ class Manager(periodic_task.PeriodicTasks):
utils.spawn_n(do_container_rebuild)
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.REBUILD)
def _do_container_rebuild(self, context, container, run):
LOG.info("start to rebuild container: %s", container.uuid)
with self._update_task_state(context, container,
@ -702,12 +710,15 @@ class Manager(periodic_task.PeriodicTasks):
def container_start(self, context, container):
@utils.synchronized(container.uuid)
def do_container_start():
self._do_container_start(context, container)
with utils.FinishAction(context, container_actions.START,
container.uuid):
self._do_container_start(context, container)
utils.spawn_n(do_container_start)
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.PAUSE)
def _do_container_pause(self, context, container):
LOG.debug('Pausing container: %s', container.uuid)
with self._update_task_state(context, container,
@ -723,7 +734,8 @@ class Manager(periodic_task.PeriodicTasks):
utils.spawn_n(do_container_pause)
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.UNPAUSE)
def _do_container_unpause(self, context, container):
LOG.debug('Unpausing container: %s', container.uuid)
with self._update_task_state(context, container,
@ -800,7 +812,8 @@ class Manager(periodic_task.PeriodicTasks):
raise
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.KILL)
def _do_container_kill(self, context, container, signal):
LOG.debug('Killing a container: %s', container.uuid)
with self._update_task_state(context, container,
@ -973,7 +986,8 @@ class Manager(periodic_task.PeriodicTasks):
'docker')
raise
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.COMMIT)
def _do_container_commit(self, context, snapshot_image, container,
repository, tag=None):
container_image_id = None
@ -1158,7 +1172,8 @@ class Manager(periodic_task.PeriodicTasks):
utils.spawn_n(do_network_detach)
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.NETWORK_DETACH)
def _do_network_detach(self, context, container, network):
LOG.debug('Detach network: %(network)s from container: %(container)s.',
{'container': container, 'network': network})
@ -1174,7 +1189,8 @@ class Manager(periodic_task.PeriodicTasks):
utils.spawn_n(do_network_attach)
@wrap_exception()
@wrap_container_event(prefix='compute')
@wrap_container_event(prefix='compute',
finish_action=container_actions.NETWORK_ATTACH)
def _do_network_attach(self, context, container, requested_network):
LOG.debug('Attach network: %(network)s to container: %(container)s.',
{'container': container, 'network': requested_network})

6
zun/db/api.py
View File

@ -850,6 +850,12 @@ def action_start(context, values):
return _get_dbdriver_instance().action_start(context, values)
@profiler.trace("db")
def action_finish(context, values):
"""Start an action for an container."""
return _get_dbdriver_instance().action_finish(context, values)
@profiler.trace("db")
def actions_get(context, uuid):
"""Get all container actions for the provided container."""

16
zun/db/sqlalchemy/api.py
View File

@ -947,6 +947,17 @@ class Connection(object):
action.save()
return action
def action_finish(self, context, values):
query = model_query(models.ContainerAction).\
filter_by(container_uuid=values['container_uuid']).\
filter_by(request_id=values['request_id']).\
filter_by(action=values['action'])
if query.update(values) != 1:
raise exception.ContainerActionNotFound(
request_id=values['request_id'],
container_uuid=values['container_uuid'])
return query.one()
def actions_get(self, context, container_uuid):
"""Get all container actions for the provided uuid."""
query = model_query(models.ContainerAction).\
@ -1022,6 +1033,7 @@ class Connection(object):
raise exception.ContainerActionNotFound(
request_id=values['request_id'],
container_uuid=values['container_uuid'])
event = model_query(models.ContainerActionEvent).\
filter_by(action_id=action['id']).\
filter_by(event=values['event']).\
@ -1034,10 +1046,6 @@ class Connection(object):
event.update(values)
event.save()
if values['result'].lower() == 'error':
action.update({'message': 'Error'})
action.save()
return event
def action_events_get(self, context, action_id):

15
zun/objects/container_action.py
View File

@ -68,10 +68,14 @@ class ContainerAction(base.ZunPersistentObject, base.ZunObject):
return values
@staticmethod
def pack_action_finish(context, container_uuid):
def pack_action_finish(context, container_uuid, action_name,
exc_val=None, exc_tb=None):
values = {'request_id': context.request_id,
'container_uuid': container_uuid,
'action': action_name,
'finish_time': timeutils.utcnow()}
if exc_tb is not None:
values['message'] = 'Error'
return values
@base.remotable_classmethod
@ -89,6 +93,15 @@ class ContainerAction(base.ZunPersistentObject, base.ZunObject):
if want_result:
return cls._from_db_object(context, cls(context), db_action)
@base.remotable_classmethod
def action_finish(cls, context, container_uuid, action_name, exc_val=None,
exc_tb=None, want_result=True):
values = cls.pack_action_finish(context, container_uuid, action_name,
exc_val=exc_val, exc_tb=exc_tb)
db_action = dbapi.action_finish(context, values)
if want_result:
return cls._from_db_object(context, cls(context), db_action)
@base.remotable_classmethod
def get_by_container_uuid(cls, context, container_uuid):
db_actions = dbapi.actions_get(context, container_uuid)

46
zun/tests/unit/compute/test_compute_manager.py
View File

@ -24,6 +24,7 @@ from zun.compute import manager
import zun.conf
from zun import objects
from zun.objects.container import Container
from zun.objects.container_action import ContainerAction
from zun.objects.container_action import ContainerActionEvent
from zun.objects.exec_instance import ExecInstance
from zun.objects.image import Image
@ -330,6 +331,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch('zun.common.utils.spawn_n')
@mock.patch.object(Container, 'save')
@mock.patch.object(VolumeMapping, 'count',
@ -346,7 +348,8 @@ class TestManager(base.TestCase):
self, mock_start, mock_create,
mock_is_volume_available, mock_attach_volume,
mock_detach_volume, mock_pull, mock_list_by_container, mock_count,
mock_save, mock_spawn_n, mock_event_finish, mock_event_start):
mock_save, mock_spawn_n, mock_action_finish, mock_event_finish,
mock_event_start):
container = Container(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_create.return_value = container
@ -377,6 +380,7 @@ class TestManager(base.TestCase):
@mock.patch.object(fake_driver, 'delete_volume')
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch('zun.common.utils.spawn_n')
@mock.patch.object(Container, 'save')
@mock.patch.object(VolumeMapping, 'count',
@ -396,8 +400,8 @@ class TestManager(base.TestCase):
mock_is_volume_available, mock_attach_volume,
mock_detach_volume, mock_pull, mock_list_by_container,
mock_list_by_volume, mock_count, mock_save,
mock_spawn_n, mock_event_finish, mock_event_start,
mock_delete_volume):
mock_spawn_n, mock_action_finish, mock_event_finish,
mock_event_start, mock_delete_volume):
mock_is_volume_available.return_value = True, False
mock_attach_volume.side_effect = [None, base.TestingException("fake")]
container = Container(self.context, **utils.get_test_container())
@ -433,6 +437,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch('zun.common.utils.spawn_n')
@mock.patch.object(Container, 'save')
@mock.patch.object(VolumeMapping, 'count',
@ -449,7 +454,7 @@ class TestManager(base.TestCase):
self, mock_pull, mock_is_volume_available,
mock_attach_volume, mock_detach_volume,
mock_list_by_container, mock_list_by_volume, mock_count,
mock_save, mock_spawn_n, mock_event_finish,
mock_save, mock_spawn_n, mock_action_finish, mock_event_finish,
mock_event_start):
container_dict = utils.get_test_container(
image='test:latest', image_driver='docker',
@ -482,6 +487,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch('zun.common.utils.spawn_n')
@mock.patch.object(Container, 'save')
@mock.patch.object(VolumeMapping, 'count',
@ -498,7 +504,7 @@ class TestManager(base.TestCase):
self, mock_pull, mock_is_volume_available,
mock_attach_volume, mock_detach_volume,
mock_list_by_container, mock_list_by_volume, mock_count,
mock_save, mock_spawn_n, mock_event_finish,
mock_save, mock_spawn_n, mock_action_finish, mock_event_finish,
mock_event_start):
container_dict = utils.get_test_container(
image='test:latest', image_driver='docker',
@ -531,6 +537,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch('zun.common.utils.spawn_n')
@mock.patch.object(Container, 'save')
@mock.patch.object(VolumeMapping, 'count',
@ -547,7 +554,7 @@ class TestManager(base.TestCase):
self, mock_pull, mock_is_volume_available,
mock_attach_volume, mock_detach_volume,
mock_list_by_container, mock_list_by_volume, mock_count,
mock_save, mock_spawn_n, mock_event_finish,
mock_save, mock_spawn_n, mock_action_finish, mock_event_finish,
mock_event_start):
container_dict = utils.get_test_container(
image='test:latest', image_driver='docker',
@ -580,6 +587,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch('zun.common.utils.spawn_n')
@mock.patch.object(Container, 'save')
@mock.patch.object(VolumeMapping, 'count',
@ -597,7 +605,7 @@ class TestManager(base.TestCase):
self, mock_create, mock_pull, mock_is_volume_available,
mock_attach_volume, mock_detach_volume,
mock_list_by_container, mock_list_by_volume, mock_count,
mock_save, mock_spawn_n,
mock_save, mock_spawn_n, mock_action_finish,
mock_event_finish, mock_event_start):
container = Container(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance',
@ -632,6 +640,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch('zun.common.utils.spawn_n')
@mock.patch.object(objects.Capsule, 'save')
@mock.patch.object(objects.CapsuleContainer, 'list_by_capsule_id')
@ -651,7 +660,8 @@ class TestManager(base.TestCase):
mock_is_volume_available, mock_attach_volume,
mock_detach_volume, mock_pull, mock_list_by_container, mock_count,
mock_init_container_list, mock_capsule_container_list,
mock_save, mock_spawn_n, mock_event_finish, mock_event_start):
mock_save, mock_spawn_n, mock_action_finish, mock_event_finish,
mock_event_start):
capsule = objects.Capsule(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_create.return_value = capsule
@ -778,6 +788,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch('zun.compute.manager.Manager._get_vol_info')
@mock.patch('zun.compute.manager.Manager._get_network_info')
@mock.patch.object(fake_driver, 'pull_image')
@ -788,6 +799,7 @@ class TestManager(base.TestCase):
def test_container_rebuild(self, mock_delete, mock_create,
mock_save, mock_check, mock_pull,
mock_get_network_info, mock_get_vol_info,
mock_action_finish,
mock_event_finish, mock_event_start):
container = Container(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
@ -803,6 +815,7 @@ class TestManager(base.TestCase):
mock_delete.assert_called_once_with(self.context, container, True)
mock_event_start.assert_called_once()
mock_event_finish.assert_called_once()
mock_action_finish.assert_called_once()
self.assertEqual(
(self.context, container.uuid, 'compute__do_container_rebuild'),
mock_event_finish.call_args[0])
@ -811,13 +824,15 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch.object(Container, 'save')
@mock.patch('zun.compute.manager.Manager._get_vol_info')
@mock.patch('zun.compute.manager.Manager._get_network_info')
@mock.patch.object(manager.Manager, '_fail_container')
def test_container_rebuild_failed(
self, mock_fail, mock_get_network_info, mock_get_vol_info,
mock_save, mock_event_finish, mock_event_start):
mock_save, mock_action_finish, mock_event_finish,
mock_event_start):
mock_get_vol_info.return_value = []
fake_exc = exception.PortNotFound(port='fake-port')
mock_get_network_info.side_effect = fake_exc
@ -829,6 +844,7 @@ class TestManager(base.TestCase):
container, str(fake_exc))
mock_event_start.assert_called_once()
mock_event_finish.assert_called_once()
mock_action_finish.assert_called_once()
self.assertEqual(
(self.context, container.uuid, 'compute__do_container_rebuild'),
mock_event_finish.call_args[0])
@ -1250,6 +1266,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch.object(fake_driver, 'upload_image_data')
@mock.patch.object(fake_driver, 'get_image')
@mock.patch.object(fake_driver, 'commit')
@ -1258,8 +1275,8 @@ class TestManager(base.TestCase):
@mock.patch.object(Container, 'save')
def test_container_commit(
self, mock_save, mock_unpause, mock_pause, mock_commit,
mock_get_image, mock_upload_image_data, mock_event_finish,
mock_event_start):
mock_get_image, mock_upload_image_data, mock_action_finish,
mock_event_finish, mock_event_start):
container = Container(self.context, **utils.get_test_container(
status=consts.PAUSED))
mock_get_image_response = mock.MagicMock()
@ -1282,6 +1299,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch.object(fake_driver, 'upload_image_data')
@mock.patch.object(fake_driver, 'get_image')
@mock.patch.object(fake_driver, 'commit')
@ -1290,8 +1308,8 @@ class TestManager(base.TestCase):
@mock.patch.object(Container, 'save')
def test_container_commit_with_pause(
self, mock_save, mock_unpause, mock_pause, mock_commit,
mock_get_image, mock_upload_image_data, mock_event_finish,
mock_event_start):
mock_get_image, mock_upload_image_data, mock_action_finish,
mock_event_finish, mock_event_start):
container = Container(self.context, **utils.get_test_container())
mock_get_image_response = mock.MagicMock()
mock_get_image_response.data = StringIO().read()
@ -1315,6 +1333,7 @@ class TestManager(base.TestCase):
@mock.patch.object(ContainerActionEvent, 'event_start')
@mock.patch.object(ContainerActionEvent, 'event_finish')
@mock.patch.object(ContainerAction, 'action_finish')
@mock.patch.object(fake_driver, 'delete_committed_image')
@mock.patch.object(fake_driver, 'commit')
@mock.patch.object(fake_driver, 'pause')
@ -1322,6 +1341,7 @@ class TestManager(base.TestCase):
@mock.patch.object(Container, 'save')
def test_container_commit_failed(self, mock_save, mock_unpause, mock_pause,
mock_commit, mock_delete,
mock_action_finish,
mock_event_finish, mock_event_start):
container = Container(self.context, **utils.get_test_container())
mock_get_image_response = mock.MagicMock()

2
zun/tests/unit/objects/test_objects.py
View File

@ -366,7 +366,7 @@ object_data = {
'QuotaClass': '1.2-4739583a70891fbc145031228fb8001e',
'ContainerPCIRequest': '1.0-b060f9f9f734bedde79a71a4d3112ee0',
'ContainerPCIRequests': '1.0-7b8f7f044661fe4e24e6949c035af2c4',
'ContainerAction': '1.2-e8c494b11ea259655256d87aef877eef',
'ContainerAction': '1.2-4ae05fe3d1576c211c2425e4db190ef2',
'ContainerActionEvent': '1.0-2974d0a6f5d4821fd4e223a88c10181a',
'Network': '1.1-26e8d37a54e5fc905ede657744a221d9',
'ExecInstance': '1.0-59464e7b96db847c0abb1e96d3cec30a',