From 8fd6f95044b1d592e9a0bd7b8682cb86000a3bc1 Mon Sep 17 00:00:00 2001 From: Chaolei Li Date: Mon, 11 Dec 2017 17:23:24 +0800 Subject: [PATCH] Add container action etcd api Add container action and container action event etcd api. In etcd, using action uuid and event uuid instead of id to get the unique action or event. Change-Id: Ia761f1a94cd2b54d371b30a7be02543a539c5cb4 --- zun/db/etcd/api.py | 193 +++++++++++++++ zun/db/etcd/models.py | 50 ++++ zun/tests/unit/db/test_container_action.py | 258 +++++++++++++++++++-- 3 files changed, 487 insertions(+), 14 deletions(-) diff --git a/zun/db/etcd/api.py b/zun/db/etcd/api.py index 3bc9d5f84..d1d31ae80 100644 --- a/zun/db/etcd/api.py +++ b/zun/db/etcd/api.py @@ -84,6 +84,10 @@ def translate_etcd_result(etcd_result, model_type): ret = models.PciDevice(data) elif model_type == 'volume_mapping': ret = models.VolumeMapping(data) + elif model_type == 'container_action': + ret = models.ContainerAction(data) + elif model_type == 'container_action_event': + ret = models.ContainerActionEvent(data) else: raise exception.InvalidParameterValue( _('The model_type value: %s is invalid.'), model_type) @@ -947,3 +951,192 @@ class EtcdAPI(object): raise return translate_etcd_result(target, 'volume_mapping') + + @lockutils.synchronized('etcd_action') + def action_start(self, context, values): + values['created_at'] = datetime.isoformat(timeutils.utcnow()) + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + action = models.ContainerAction(values) + try: + action.save() + except Exception: + raise + return action + + def _actions_get(self, context, container_uuid, filters=None): + action_path = '/container_actions/' + container_uuid + + try: + res = getattr(self.client.read(action_path), 'children', None) + except etcd.EtcdKeyNotFound: + return [] + except Exception as e: + LOG.error( + "Error occurred while reading from etcd server: %s", + six.text_type(e)) + raise + + actions = [] + for c in res: + if c.value is not None: + actions.append(translate_etcd_result(c, 'container_action')) + filters = self._add_project_filters(context, filters) + filtered_actions = self._filter_resources(actions, filters) + sorted_actions = self._process_list_result(filtered_actions, + sort_key='created_at') + # Actions need descending order of created_at. + sorted_actions.reverse() + return sorted_actions + + def actions_get(self, context, container_uuid): + return self._actions_get(context, container_uuid) + + def _action_get_by_request_id(self, context, container_uuid, request_id): + filters = {'request_id': request_id} + actions = self._actions_get(context, container_uuid, filters=filters) + if not actions: + return None + return actions[0] + + def action_get_by_request_id(self, context, container_uuid, request_id): + return self._action_get_by_request_id(context, container_uuid, + request_id) + + @lockutils.synchronized('etcd_action') + def action_event_start(self, context, values): + """Start an event on a container action.""" + action = self._action_get_by_request_id(context, + values['container_uuid'], + values['request_id']) + + # When zun-compute restarts, the request_id was different with + # request_id recorded in ContainerAction, so we can't get the original + # recode according to request_id. Try to get the last created action + # so that init_container can continue to finish the recovery action. + if not action and not context.project_id: + actions = self._actions_get(context, values['container_uuid']) + if not actions: + action = actions[0] + + if not action: + raise exception.ContainerActionNotFound( + request_id=values['request_id'], + container_uuid=values['container_uuid']) + + values['action_id'] = action['id'] + values['action_uuid'] = action['uuid'] + + values['created_at'] = datetime.isoformat(timeutils.utcnow()) + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + event = models.ContainerActionEvent(values) + try: + event.save() + except Exception: + raise + return event + + def _action_events_get(self, context, action_uuid, filters=None): + event_path = '/container_actions_events/' + action_uuid + + try: + res = getattr(self.client.read(event_path), 'children', None) + except etcd.EtcdKeyNotFound: + return [] + except Exception as e: + LOG.error( + "Error occurred while reading from etcd server: %s", + six.text_type(e)) + raise + + events = [] + for c in res: + if c.value is not None: + events.append(translate_etcd_result( + c, 'container_action_event')) + + filters = filters or {} + filtered_events = self._filter_resources(events, filters) + sorted_events = self._process_list_result(filtered_events, + sort_key='created_at') + # Events need descending order of created_at. + sorted_events.reverse() + return sorted_events + + def _get_event_by_name(self, context, action_uuid, event_name): + filters = {'event': event_name} + events = self._action_events_get(context, action_uuid, filters) + if not events: + return None + return events[0] + + @lockutils.synchronized('etcd_action') + def action_event_finish(self, context, values): + """Finish an event on a container action.""" + action = self._action_get_by_request_id(context, + values['container_uuid'], + values['request_id']) + + # When zun-compute restarts, the request_id was different with + # request_id recorded in ContainerAction, so we can't get the original + # recode according to request_id. Try to get the last created action + # so that init_container can continue to finish the recovery action. + if not action and not context.project_id: + actions = self._actions_get(context, values['container_uuid']) + if not actions: + action = actions[0] + + if not action: + raise exception.ContainerActionNotFound( + request_id=values['request_id'], + container_uuid=values['container_uuid']) + + event = self._get_event_by_name(context, action['uuid'], + values['event']) + + if not event: + raise exception.ContainerActionEventNotFound( + action_id=action['uuid'], event=values['event']) + + try: + target_path = '/container_actions_events/{0}/{1}'.\ + format(action['uuid'], event['uuid']) + target = self.client.read(target_path) + target_values = json.loads(target.value) + target_values.update(values) + target.value = json.dump_as_bytes(target_values) + self.client.update(target) + except etcd.EtcdKeyNotFound: + raise exception.ContainerActionEventNotFound( + action_id=action['uuid'], event=values['event']) + except Exception as e: + LOG.error('Error occurred while updating action event: %s', + six.text_type(e)) + raise + + if values['result'].lower() == 'error': + try: + target_path = '/container_actions/{0}/{1}'.\ + format(action['container_uuid'], action['uuid']) + target = self.client.read(target_path) + target_values = json.loads(target.value) + target_values.update({'message': 'Error'}) + target.value = json.dump_as_bytes(target_values) + self.client.update(target) + except etcd.EtcdKeyNotFound: + raise exception.ContainerActionNotFound( + request_id=action['request_id'], + container_uuid=action['container_uuid']) + except Exception as e: + LOG.error('Error occurred while updating action : %s', + six.text_type(e)) + raise + + return event + + def action_events_get(self, context, action_id): + events = self._action_events_get(context, action_id) + return events diff --git a/zun/db/etcd/models.py b/zun/db/etcd/models.py index b27d40dbe..7719b6aa3 100644 --- a/zun/db/etcd/models.py +++ b/zun/db/etcd/models.py @@ -312,3 +312,53 @@ class VolumeMapping(Base): @classmethod def fields(cls): return cls._fields + + +class ContainerAction(Base): + """Represents a container action. + + The intention is that there will only be one of these pre user request. A + lookup by(container_uuid, request_id) should always return a single result. + """ + _path = '/container_actions' + + _fields = list(objects.ContainerAction.fields) + ['uuid'] + + def __init__(self, action_data): + self.path = ContainerAction.path(action_data['container_uuid']) + for f in ContainerAction.fields(): + setattr(self, f, None) + self.id = 1 + self.update(action_data) + + @classmethod + def path(cls, container_uuid): + return cls._path + '/' + container_uuid + + @classmethod + def fields(cls): + return cls._fields + + +class ContainerActionEvent(Base): + """Track events that occur during an ContainerAction.""" + + _path = '/container_actions_events' + + _fields = list(objects.ContainerActionEvent.fields) + ['action_uuid', + 'uuid'] + + def __init__(self, event_data): + self.path = ContainerActionEvent.path(event_data['action_uuid']) + for f in ContainerActionEvent.fields(): + setattr(self, f, None) + self.id = 1 + self.update(event_data) + + @classmethod + def path(cls, action_uuid): + return cls._path + '/' + action_uuid + + @classmethod + def fields(cls): + return cls._fields diff --git a/zun/tests/unit/db/test_container_action.py b/zun/tests/unit/db/test_container_action.py index 8052ecbde..3cd960a57 100644 --- a/zun/tests/unit/db/test_container_action.py +++ b/zun/tests/unit/db/test_container_action.py @@ -11,8 +11,11 @@ # under the License. """Tests for manipulating Container Actions via the DB API""" -import datetime - +from datetime import datetime +from datetime import timedelta +import etcd +from etcd import Client as etcd_client +import mock from oslo_config import cfg from oslo_utils import timeutils from oslo_utils import uuidutils @@ -20,26 +23,26 @@ from oslo_utils import uuidutils from zun.common import exception import zun.conf from zun.db import api as dbapi +from zun.db.etcd.api import EtcdAPI as etcd_api from zun.tests.unit.db import base from zun.tests.unit.db import utils +from zun.tests.unit.db.utils import FakeEtcdMultipleResult +from zun.tests.unit.db.utils import FakeEtcdResult CONF = zun.conf.CONF -class DbContainerActionTestCase(base.DbTestCase, - base.ModelsObjectComparatorMixin): +class _DBContainerActionBase(base.DbTestCase, + base.ModelsObjectComparatorMixin): + IGNORED_FIELDS = [ 'id', 'created_at', 'updated_at', + 'details' ] - def setUp(self): - cfg.CONF.set_override('db_type', 'sql') - super(DbContainerActionTestCase, self).setUp() - def _create_action_values(self, uuid, action='create_container'): - utils.create_test_container(context=self.context, name='cont1', uuid=uuid) @@ -60,8 +63,7 @@ class DbContainerActionTestCase(base.DbTestCase, 'event': event, 'container_uuid': uuid, 'request_id': self.context.request_id, - 'start_time': timeutils.utcnow(), - 'details': 'fake-details', + 'start_time': timeutils.utcnow() } if extra is not None: values.update(extra) @@ -80,6 +82,13 @@ class DbContainerActionTestCase(base.DbTestCase, self._assertEqualObjects(event, events[0], ['container_uuid', 'request_id']) + +class DbContainerActionTestCase(_DBContainerActionBase): + + def setUp(self): + cfg.CONF.set_override('db_type', 'sql') + super(DbContainerActionTestCase, self).setUp() + def test_container_action_start(self): """Create a container action.""" uuid = uuidutils.generate_uuid() @@ -166,7 +175,7 @@ class DbContainerActionTestCase(base.DbTestCase, self._create_event_values(uuid)) event_values = { - 'finish_time': timeutils.utcnow() + datetime.timedelta(seconds=5), + 'finish_time': timeutils.utcnow() + timedelta(seconds=5), 'result': 'Success' } @@ -182,7 +191,7 @@ class DbContainerActionTestCase(base.DbTestCase, uuid = uuidutils.generate_uuid() event_values = { - 'finish_time': timeutils.utcnow() + datetime.timedelta(seconds=5), + 'finish_time': timeutils.utcnow() + timedelta(seconds=5), 'result': 'Success' } event_values = self._create_event_values(uuid, extra=event_values) @@ -202,7 +211,7 @@ class DbContainerActionTestCase(base.DbTestCase, } extra2 = { - 'created_at': timeutils.utcnow() + datetime.timedelta(seconds=5) + 'created_at': timeutils.utcnow() + timedelta(seconds=5) } event_val1 = self._create_event_values(uuid1, 'fake1', extra=extra1) @@ -219,3 +228,224 @@ class DbContainerActionTestCase(base.DbTestCase, self._assertEqualOrderedListOfObjects([event3, event2, event1], events, ['container_uuid', 'request_id']) + + +class EtcdDbContainerActionTestCase(_DBContainerActionBase): + + def setUp(self): + cfg.CONF.set_override('db_type', 'etcd') + super(EtcdDbContainerActionTestCase, self).setUp() + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_action_start(self, mock_write, mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + action_values = self._create_action_values(uuid) + action = dbapi.action_start(self.context, action_values) + + ignored_keys = self.IGNORED_FIELDS + ['finish_time', 'uuid'] + + self._assertEqualObjects(action_values, action.as_dict(), ignored_keys) + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult( + [action.as_dict()]) + action['start_time'] = datetime.isoformat(action['start_time']) + self._assertActionSaved(action.as_dict(), uuid) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_actions_get_by_container(self, mock_write, mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid1 = uuidutils.generate_uuid() + + expected = [] + + action_values = self._create_action_values(uuid1) + action = dbapi.action_start(self.context, action_values) + action['start_time'] = datetime.isoformat(action['start_time']) + expected.append(action) + + action_values['action'] = 'test-action' + action = dbapi.action_start(self.context, action_values) + action['start_time'] = datetime.isoformat(action['start_time']) + expected.append(action) + + # Create an other container action. + uuid2 = uuidutils.generate_uuid() + action_values = self._create_action_values(uuid2, 'test-action') + dbapi.action_start(self.context, action_values) + + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult( + expected) + actions = dbapi.actions_get(self.context, uuid1) + self._assertEqualListsOfObjects(expected, actions) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_action_get_by_container_and_request(self, mock_write, + mock_read): + """Ensure we can get an action by container UUID and request_id""" + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid1 = uuidutils.generate_uuid() + + action_values = self._create_action_values(uuid1) + action = dbapi.action_start(self.context, action_values) + request_id = action_values['request_id'] + + # An other action using a different req id + action_values['action'] = 'test-action' + action_values['request_id'] = 'req-00000000-7522-4d99-7ff-111111111111' + dbapi.action_start(self.context, action_values) + + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([action]) + action = dbapi.action_get_by_request_id(self.context, uuid1, + request_id) + self.assertEqual('create_container', action['action']) + self.assertEqual(self.context.request_id, action['request_id']) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + @mock.patch.object(etcd_api, '_action_get_by_request_id') + def test_container_action_event_start(self, mock__action_get_by_request_id, + mock_write, mock_read): + """Create a container action event.""" + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + + action_values = self._create_action_values(uuid) + action = dbapi.action_start(self.context, action_values) + + event_values = self._create_event_values(uuid) + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event = dbapi.action_event_start(self.context, event_values) + + ignored_keys = self.IGNORED_FIELDS + ['finish_time', 'traceback', + 'result', 'action_uuid', + 'request_id', 'container_uuid', + 'uuid'] + self._assertEqualObjects(event_values, event, ignored_keys) + + event['start_time'] = datetime.isoformat(event['start_time']) + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([event]) + self._assertActionEventSaved(event, action['uuid']) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_action_event_start_without_action(self, mock_write, + mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + + event_values = self._create_event_values(uuid) + self.assertRaises(exception.ContainerActionNotFound, + dbapi.action_event_start, self.context, event_values) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + @mock.patch.object(etcd_client, 'update') + @mock.patch.object(etcd_api, '_action_get_by_request_id') + @mock.patch.object(etcd_api, '_get_event_by_name') + def test_container_action_event_finish_success( + self, mock_get_event_by_name, mock__action_get_by_request_id, + mock_update, mock_write, mock_read): + """Finish a container action event.""" + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + + action = dbapi.action_start(self.context, + self._create_action_values(uuid)) + + event_values = self._create_event_values(uuid) + event_values['action_uuid'] = action['uuid'] + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event = dbapi.action_event_start(self.context, event_values) + + event_values = { + 'finish_time': timeutils.utcnow() + timedelta(seconds=5), + 'result': 'Success' + } + + event_values = self._create_event_values(uuid, extra=event_values) + + mock__action_get_by_request_id.return_value = action + mock_get_event_by_name.return_value = event + mock_read.side_effect = lambda *args: FakeEtcdResult(event) + event = dbapi.action_event_finish(self.context, event_values) + + event['start_time'] = datetime.isoformat(event['start_time']) + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([event]) + self._assertActionEventSaved(event, action['uuid']) + + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult([action]) + action = dbapi.action_get_by_request_id(self.context, uuid, + self.context.request_id) + self.assertNotEqual('Error', action['message']) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + def test_container_action_event_finish_without_action(self, mock_write, + mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid = uuidutils.generate_uuid() + + event_values = { + 'finish_time': timeutils.utcnow() + timedelta(seconds=5), + 'result': 'Success' + } + event_values = self._create_event_values(uuid, extra=event_values) + self.assertRaises(exception.ContainerActionNotFound, + dbapi.action_event_finish, + self.context, event_values) + + @mock.patch.object(etcd_client, 'read') + @mock.patch.object(etcd_client, 'write') + @mock.patch.object(etcd_api, '_action_get_by_request_id') + def test_container_action_events_get_in_order( + self, mock__action_get_by_request_id, mock_write, mock_read): + mock_read.side_effect = etcd.EtcdKeyNotFound + uuid1 = uuidutils.generate_uuid() + + action = dbapi.action_start(self.context, + self._create_action_values(uuid1)) + + extra1 = { + 'action_uuid': action['uuid'], + 'created_at': timeutils.utcnow() + } + + extra2 = { + 'action_uuid': action['uuid'], + 'created_at': timeutils.utcnow() + timedelta(seconds=5) + } + + event_val1 = self._create_event_values(uuid1, 'fake1', extra=extra1) + event_val2 = self._create_event_values(uuid1, 'fake2', extra=extra1) + event_val3 = self._create_event_values(uuid1, 'fake3', extra=extra2) + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event1 = dbapi.action_event_start(self.context, event_val1) + event1['start_time'] = datetime.isoformat(event1['start_time']) + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event2 = dbapi.action_event_start(self.context, event_val2) + event2['start_time'] = datetime.isoformat(event2['start_time']) + + mock__action_get_by_request_id.return_value = action + mock_read.side_effect = etcd.EtcdKeyNotFound + event3 = dbapi.action_event_start(self.context, event_val3) + event3['start_time'] = datetime.isoformat(event3['start_time']) + + mock_read.side_effect = lambda *args: FakeEtcdMultipleResult( + [event1, event2, event3]) + events = dbapi.action_events_get(self.context, action['uuid']) + + self.assertEqual(3, len(events)) + + self._assertEqualOrderedListOfObjects([event3, event2, event1], events, + ['container_uuid', 'request_id'])