diff --git a/zun/db/etcd/api.py b/zun/db/etcd/api.py index 3bc9d5f84..d1d31ae80 100644 --- a/zun/db/etcd/api.py +++ b/zun/db/etcd/api.py @@ -84,6 +84,10 @@ def translate_etcd_result(etcd_result, model_type): ret = models.PciDevice(data) elif model_type == 'volume_mapping': ret = models.VolumeMapping(data) + elif model_type == 'container_action': + ret = models.ContainerAction(data) + elif model_type == 'container_action_event': + ret = models.ContainerActionEvent(data) else: raise exception.InvalidParameterValue( _('The model_type value: %s is invalid.'), model_type) @@ -947,3 +951,192 @@ class EtcdAPI(object): raise return translate_etcd_result(target, 'volume_mapping') + + @lockutils.synchronized('etcd_action') + def action_start(self, context, values): + values['created_at'] = datetime.isoformat(timeutils.utcnow()) + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + action = models.ContainerAction(values) + try: + action.save() + except Exception: + raise + return action + + def _actions_get(self, context, container_uuid, filters=None): + action_path = '/container_actions/' + container_uuid + + try: + res = getattr(self.client.read(action_path), 'children', None) + except etcd.EtcdKeyNotFound: + return [] + except Exception as e: + LOG.error( + "Error occurred while reading from etcd server: %s", + six.text_type(e)) + raise + + actions = [] + for c in res: + if c.value is not None: + actions.append(translate_etcd_result(c, 'container_action')) + filters = self._add_project_filters(context, filters) + filtered_actions = self._filter_resources(actions, filters) + sorted_actions = self._process_list_result(filtered_actions, + sort_key='created_at') + # Actions need descending order of created_at. + sorted_actions.reverse() + return sorted_actions + + def actions_get(self, context, container_uuid): + return self._actions_get(context, container_uuid) + + def _action_get_by_request_id(self, context, container_uuid, request_id): + filters = {'request_id': request_id} + actions = self._actions_get(context, container_uuid, filters=filters) + if not actions: + return None + return actions[0] + + def action_get_by_request_id(self, context, container_uuid, request_id): + return self._action_get_by_request_id(context, container_uuid, + request_id) + + @lockutils.synchronized('etcd_action') + def action_event_start(self, context, values): + """Start an event on a container action.""" + action = self._action_get_by_request_id(context, + values['container_uuid'], + values['request_id']) + + # When zun-compute restarts, the request_id was different with + # request_id recorded in ContainerAction, so we can't get the original + # recode according to request_id. Try to get the last created action + # so that init_container can continue to finish the recovery action. + if not action and not context.project_id: + actions = self._actions_get(context, values['container_uuid']) + if not actions: + action = actions[0] + + if not action: + raise exception.ContainerActionNotFound( + request_id=values['request_id'], + container_uuid=values['container_uuid']) + + values['action_id'] = action['id'] + values['action_uuid'] = action['uuid'] + + values['created_at'] = datetime.isoformat(timeutils.utcnow()) + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + event = models.ContainerActionEvent(values) + try: + event.save() + except Exception: + raise + return event + + def _action_events_get(self, context, action_uuid, filters=None): + event_path = '/container_actions_events/' + action_uuid + + try: + res = getattr(self.client.read(event_path), 'children', None) + except etcd.EtcdKeyNotFound: + return [] + except Exception as e: + LOG.error( + "Error occurred while reading from etcd server: %s", + six.text_type(e)) + raise + + events = [] + for c in res: + if c.value is not None: + events.append(translate_etcd_result( + c, 'container_action_event')) + + filters = filters or {} + filtered_events = self._filter_resources(events, filters) + sorted_events = self._process_list_result(filtered_events, + sort_key='created_at') + # Events need descending order of created_at. + sorted_events.reverse() + return sorted_events + + def _get_event_by_name(self, context, action_uuid, event_name): + filters = {'event': event_name} + events = self._action_events_get(context, action_uuid, filters) + if not events: + return None + return events[0] + + @lockutils.synchronized('etcd_action') + def action_event_finish(self, context, values): + """Finish an event on a container action.""" + action = self._action_get_by_request_id(context, + values['container_uuid'], + values['request_id']) + + # When zun-compute restarts, the request_id was different with + # request_id recorded in ContainerAction, so we can't get the original + # recode according to request_id. Try to get the last created action + # so that init_container can continue to finish the recovery action. + if not action and not context.project_id: + actions = self._actions_get(context, values['container_uuid']) + if not actions: + action = actions[0] + + if not action: + raise exception.ContainerActionNotFound( + request_id=values['request_id'], + container_uuid=values['container_uuid']) + + event = self._get_event_by_name(context, action['uuid'], + values['event']) + + if not event: + raise exception.ContainerActionEventNotFound( + action_id=action['uuid'], event=values['event']) + + try: + target_path = '/container_actions_events/{0}/{1}'.\ + format(action['uuid'], event['uuid']) + target = self.client.read(target_path) + target_values = json.loads(target.value) + target_values.update(values) + target.value = json.dump_as_bytes(target_values) + self.client.update(target) + except etcd.EtcdKeyNotFound: + raise exception.ContainerActionEventNotFound( + action_id=action['uuid'], event=values['event']) + except Exception as e: + LOG.error('Error occurred while updating action event: %s', + six.text_type(e)) + raise + + if values['result'].lower() == 'error': + try: + target_path = '/container_actions/{0}/{1}'.\ + format(action['container_uuid'], action['uuid']) + target = self.client.read(target_path) + target_values = json.loads(target.value) + target_values.update({'message': 'Error'}) + target.value = json.dump_as_bytes(target_values) + self.client.update(target) + except etcd.EtcdKeyNotFound: + raise exception.ContainerActionNotFound( + request_id=action['request_id'], + container_uuid=action['container_uuid']) + except Exception as e: + LOG.error('Error occurred while updating action : %s', + six.text_type(e)) + raise + + return event + + def action_events_get(self, context, action_id): + events = self._action_events_get(context, action_id) + return events diff --git a/zun/db/etcd/models.py b/zun/db/etcd/models.py index b27d40dbe..7719b6aa3 100644 --- a/zun/db/etcd/models.py +++ b/zun/db/etcd/models.py @@ -312,3 +312,53 @@ class VolumeMapping(Base): @classmethod def fields(cls): return cls._fields + + +class ContainerAction(Base): + """Represents a container action. + + The intention is that there will only be one of these pre user request. A + lookup by(container_uuid, request_id) should always return a single result. + """ + _path = '/container_actions' + + _fields = list(objects.ContainerAction.fields) + ['uuid'] + + def __init__(self, action_data): + self.path = ContainerAction.path(action_data['container_uuid']) + for f in ContainerAction.fields(): + setattr(self, f, None) + self.id = 1 + self.update(action_data) + + @classmethod + def path(cls, container_uuid): + return cls._path + '/' + container_uuid + + @classmethod + def fields(cls): + return cls._fields + + +class ContainerActionEvent(Base): + """Track events that occur during an ContainerAction.""" + + _path = '/container_actions_events' + + _fields = list(objects.ContainerActionEvent.fields) + ['action_uuid', + 'uuid'] + + def __init__(self, event_data): + self.path = ContainerActionEvent.path(event_data['action_uuid']) + for f in ContainerActionEvent.fields(): + setattr(self, f, None) + self.id = 1 + self.update(event_data) + + @classmethod + def path(cls, action_uuid): + return cls._path + '/' + action_uuid + + @classmethod + def fields(cls): + return cls._fields diff --git a/zun/tests/unit/db/test_container_action.py b/zun/tests/unit/db/test_container_action.py index 8052ecbde..3cd960a57 100644 --- a/zun/tests/unit/db/test_container_action.py +++ b/zun/tests/unit/db/test_container_action.py @@ -11,8 +11,11 @@ # under the License. """Tests for manipulating Container Actions via the DB API""" -import datetime - +from datetime import datetime +from datetime import timedelta +import etcd +from etcd import Client as etcd_client +import mock from oslo_config import cfg from oslo_utils import timeutils from oslo_utils import uuidutils @@ -20,26 +23,26 @@ from oslo_utils import uuidutils from zun.common import exception import zun.conf from zun.db import api as dbapi +from zun.db.etcd.api import EtcdAPI as etcd_api from zun.tests.unit.db import base from zun.tests.unit.db import utils +from zun.tests.unit.db.utils import FakeEtcdMultipleResult +from zun.tests.unit.db.utils import FakeEtcdResult CONF = zun.conf.CONF -class DbContainerActionTestCase(base.DbTestCase, - base.ModelsObjectComparatorMixin): +class _DBContainerActionBase(base.DbTestCase, + base.ModelsObjectComparatorMixin): + IGNORED_FIELDS = [ 'id', 'created_at', 'updated_at', + 'details' ] - def setUp(self): - cfg.CONF.set_override('db_type', 'sql') - super(DbContainerActionTestCase, self).setUp() - def _create_action_values(self, uuid, action='create_container'): - utils.create_test_container(context=self.context, name='cont1', uuid=uuid) @@ -60,8 +63,7 @@ class DbContainerActionTestCase(base.DbTestCase, 'event': event, 'container_uuid': uuid, 'request_id': self.context.request_id, - 'start_time': timeutils.utcnow(), - 'details': 'fake-details', + 'start_time': timeutils.utcnow() } if extra is not None: values.update(extra) @@ -80,6 +82,13 @@ class DbContainerActionTestCase(base.DbTestCase, self._assertEqualObjects(event, events[0], ['container_uuid', 'request_id']) + +class DbContainerActionTestCase(_DBContainerActionBase): + + def setUp(self): + cfg.CONF.set_override('db_type', 'sql') + super(DbContainerActionTestCase, self).setUp() + def test_container_action_start(self): """Create a container action.""" uuid = uuidutils.generate_uuid() @@ -166,7 +175,7 @@ class DbContainerActionTestCase(base.DbTestCase, self._create_event_values(uuid)) event_values = { - 'finish_time': timeutils.utcnow() + datetime.timedelta(seconds=5), + 'finish_time': timeutils.utcnow() + timedelta(seconds=5), 'result': 'Success' } @@ -182,7 +191,7 @@ class DbContainerActionTestCase(base.DbTestCase, uuid = uuidutils.generate_uuid() event_values = { - 'finish_time': timeutils.utcnow() + datetime.timedelta(seconds=5), + 'finish_time': timeutils.utcnow() + timedelta(seconds=5), 'result': 'Success' } event_values = self._create_event_values(uuid, extra=event_values) @@ -202,7 +211,7 @@ class DbContainerActionTestCase(base.DbTestCase, } extra2 = { - 'created_at': timeutils.utcnow() + datetime.timedelta(seconds=5) + 'created_at': timeutils.utcnow() + timedelta(seconds=5) } event_val1 = self._create_event_values(uuid1, 'fake1', extra=extra1) @@ -219,3 +228,224 @@ class DbContainerActionTestCase(base.DbTestCase, self._assertEqualOrderedListOfObjects([event3, event2, event1], events, ['container_uuid', 'request_id']) + + +class EtcdDbContainerActionTestCase(_DBContainerActionBase): + + def setUp(self): + cfg.CONF.set_override('db_type', 'etcd') + super(EtcdDbContainerActionTestCase, self).setUp() + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_action_start(self, mock_write, mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + action_values = self._create_action_values(uuid) + action = dbapi.action_start(self.context, action_values) + + ignored_keys = self.IGNORED_FIELDS + ['finish_time', 'uuid'] + + self._assertEqualObjects(action_values, action.as_dict(), ignored_keys) + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult( + [action.as_dict()]) + action['start_time'] = datetime.isoformat(action['start_time']) + self._assertActionSaved(action.as_dict(), uuid) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_actions_get_by_container(self, mock_write, mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid1 = uuidutils.generate_uuid() + + expected = [] + + action_values = self._create_action_values(uuid1) + action = dbapi.action_start(self.context, action_values) + action['start_time'] = datetime.isoformat(action['start_time']) + expected.append(action) + + action_values['action'] = 'test-action' + action = dbapi.action_start(self.context, action_values) + action['start_time'] = datetime.isoformat(action['start_time']) + expected.append(action) + + # Create an other container action. + uuid2 = uuidutils.generate_uuid() + action_values = self._create_action_values(uuid2, 'test-action') + dbapi.action_start(self.context, action_values) + + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult( + expected) + actions = dbapi.actions_get(self.context, uuid1) + self._assertEqualListsOfObjects(expected, actions) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_action_get_by_container_and_request(self, mock_write, + mock_read): + """Ensure we can get an action by container UUID and request_id""" + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid1 = uuidutils.generate_uuid() + + action_values = self._create_action_values(uuid1) + action = dbapi.action_start(self.context, action_values) + request_id = action_values['request_id'] + + # An other action using a different req id + action_values['action'] = 'test-action' + action_values['request_id'] = 'req-00000000-7522-4d99-7ff-111111111111' + dbapi.action_start(self.context, action_values) + + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([action]) + action = dbapi.action_get_by_request_id(self.context, uuid1, + request_id) + self.assertEqual('create_container', action['action']) + self.assertEqual(self.context.request_id, action['request_id']) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + @mock.patch.object(etcd_api, '_action_get_by_request_id') + def test_container_action_event_start(self, mock__action_get_by_request_id, + mock_write, mock_read): + """Create a container action event.""" + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + + action_values = self._create_action_values(uuid) + action = dbapi.action_start(self.context, action_values) + + event_values = self._create_event_values(uuid) + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event = dbapi.action_event_start(self.context, event_values) + + ignored_keys = self.IGNORED_FIELDS + ['finish_time', 'traceback', + 'result', 'action_uuid', + 'request_id', 'container_uuid', + 'uuid'] + self._assertEqualObjects(event_values, event, ignored_keys) + + event['start_time'] = datetime.isoformat(event['start_time']) + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([event]) + self._assertActionEventSaved(event, action['uuid']) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_action_event_start_without_action(self, mock_write, + mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + + event_values = self._create_event_values(uuid) + self.assertRaises(exception.ContainerActionNotFound, + dbapi.action_event_start, self.context, event_values) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + @mock.patch.object(etcd_client, 'update') + @mock.patch.object(etcd_api, '_action_get_by_request_id') + @mock.patch.object(etcd_api, '_get_event_by_name') + def test_container_action_event_finish_success( + self, mock_get_event_by_name, mock__action_get_by_request_id, + mock_update, mock_write, mock_read): + """Finish a container action event.""" + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + + action = dbapi.action_start(self.context, + self._create_action_values(uuid)) + + event_values = self._create_event_values(uuid) + event_values['action_uuid'] = action['uuid'] + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event = dbapi.action_event_start(self.context, event_values) + + event_values = { + 'finish_time': timeutils.utcnow() + timedelta(seconds=5), + 'result': 'Success' + } + + event_values = self._create_event_values(uuid, extra=event_values) + + mock__action_get_by_request_id.return_value = action + mock_get_event_by_name.return_value = event + mock_read.side_effect = lambda *args: FakeEtcdResult(event) + event = dbapi.action_event_finish(self.context, event_values) + + event['start_time'] = datetime.isoformat(event['start_time']) + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([event]) + self._assertActionEventSaved(event, action['uuid']) + + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([action]) + action = dbapi.action_get_by_request_id(self.context, uuid, + self.context.request_id) + self.assertNotEqual('Error', action['message']) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_action_event_finish_without_action(self, mock_write, + mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + + event_values = { + 'finish_time': timeutils.utcnow() + timedelta(seconds=5), + 'result': 'Success' + } + event_values = self._create_event_values(uuid, extra=event_values) + self.assertRaises(exception.ContainerActionNotFound, + dbapi.action_event_finish, + self.context, event_values) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + @mock.patch.object(etcd_api, '_action_get_by_request_id') + def test_container_action_events_get_in_order( + self, mock__action_get_by_request_id, mock_write, mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid1 = uuidutils.generate_uuid() + + action = dbapi.action_start(self.context, + self._create_action_values(uuid1)) + + extra1 = { + 'action_uuid': action['uuid'], + 'created_at': timeutils.utcnow() + } + + extra2 = { + 'action_uuid': action['uuid'], + 'created_at': timeutils.utcnow() + timedelta(seconds=5) + } + + event_val1 = self._create_event_values(uuid1, 'fake1', extra=extra1) + event_val2 = self._create_event_values(uuid1, 'fake2', extra=extra1) + event_val3 = self._create_event_values(uuid1, 'fake3', extra=extra2) + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event1 = dbapi.action_event_start(self.context, event_val1) + event1['start_time'] = datetime.isoformat(event1['start_time']) + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event2 = dbapi.action_event_start(self.context, event_val2) + event2['start_time'] = datetime.isoformat(event2['start_time']) + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event3 = dbapi.action_event_start(self.context, event_val3) + event3['start_time'] = datetime.isoformat(event3['start_time']) + + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult( + [event1, event2, event3]) + events = dbapi.action_events_get(self.context, action['uuid']) + + self.assertEqual(3, len(events)) + + self._assertEqualOrderedListOfObjects([event3, event2, event1], events, + ['container_uuid', 'request_id'])