From 96bd39afad79c98d903de2cb37313f8dab7647d7 Mon Sep 17 00:00:00 2001 From: Wenzhi Yu Date: Tue, 20 Dec 2016 13:44:22 +0800 Subject: [PATCH] Enable etcd DB backend testing pipeline As disussed in project plan, we decided to adopt etcd to store container data which is faster and more flexible than mysql. This commit enable etcd DB backend testing pipeline in Zun and adjust the code correspondingly: 1. Set the value of 'db_type' to 'etcd' when runing etcd pipeline: 'gate-zun-devstack-dsvm-docker-etcd-nv'. 2. Drop the 'get_XXX_by_id' methods from etcd API since 'id' attribute makes no sense in etcd data model. 3. Adjust DB related test cases. 4. Make etcd write request thead-safe. After the etcd related code been well tested, we'll change the default DB type from 'mysql' to 'etcd'. Part of blueprint etcd-db-driver Change-Id: I06ac9b719ce93898cd4f67f1c7bdc2e0803e086e --- devstack/gate_hook.sh | 7 ++ devstack/lib/zun | 6 ++ zun/api/controllers/v1/containers.py | 2 +- zun/common/singleton.py | 10 ++- zun/compute/manager.py | 45 +++++------ zun/container/docker/driver.py | 4 +- zun/container/driver.py | 2 +- zun/db/etcd/api.py | 73 ++++++++---------- zun/db/etcd/models.py | 13 +++- zun/tests/unit/api/base.py | 5 ++ .../api/controllers/v1/test_containers.py | 4 +- .../unit/compute/test_compute_manager.py | 47 +++++++----- zun/tests/unit/container/base.py | 76 ++++++++++--------- .../container/docker/test_docker_driver.py | 24 +++--- zun/tests/unit/db/test_container.py | 20 +---- zun/tests/unit/db/test_image.py | 4 + zun/tests/unit/db/test_zun_service.py | 6 +- zun/tests/unit/objects/test_container.py | 18 +++-- 18 files changed, 198 insertions(+), 168 deletions(-) diff --git a/devstack/gate_hook.sh b/devstack/gate_hook.sh index 7f0a350b9..5023435ca 100755 --- a/devstack/gate_hook.sh +++ b/devstack/gate_hook.sh @@ -19,6 +19,7 @@ # maintain if we want to change devstack config settings in future. driver=$1 +db=$2 if [ "$driver" = "docker" ]; then export DEVSTACK_LOCAL_CONFIG+=$'\n'"ZUN_DRIVER=docker" @@ -27,4 +28,10 @@ elif [ "$driver" = "nova-docker" ]; then export DEVSTACK_LOCAL_CONFIG+=$'\n'"IP_VERSION=4" fi +if [ "$db" = "etcd" ]; then + export DEVSTACK_LOCAL_CONFIG+=$'\n'"ZUN_DB_TYPE=etcd" +elif [ "$db" = "sql" ]; then + export DEVSTACK_LOCAL_CONFIG+=$'\n'"ZUN_DB_TYPE=sql" +fi + $BASE/new/devstack-gate/devstack-vm-gate.sh diff --git a/devstack/lib/zun b/devstack/lib/zun index 5e53a38e4..27eb138f1 100644 --- a/devstack/lib/zun +++ b/devstack/lib/zun @@ -66,6 +66,7 @@ fi DOCKER_GROUP=${DOCKER_GROUP:-docker} ZUN_DRIVER=${ZUN_DRIVER:-docker} +ZUN_DB_TYPE=${ZUN_DB_TYPE:-sql} ETCD_VERSION=v3.0.13 if is_ubuntu; then @@ -190,6 +191,11 @@ function create_zun_conf { elif [[ ${ZUN_DRIVER} == "nova-docker" ]]; then iniset $ZUN_CONF DEFAULT container_driver docker.driver.NovaDockerDriver fi + if [[ ${ZUN_DB_TYPE} == "etcd" ]]; then + iniset $ZUN_CONF DEFAULT db_type etcd + elif [[ ${ZUN_DB_TYPE} == "sql" ]]; then + iniset $ZUN_CONF DEFAULT db_type sql + fi iniset $ZUN_CONF DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL" iniset $ZUN_CONF oslo_messaging_rabbit rabbit_userid $RABBIT_USERID iniset $ZUN_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD diff --git a/zun/api/controllers/v1/containers.py b/zun/api/controllers/v1/containers.py index 116743d9f..515c6a4cb 100644 --- a/zun/api/controllers/v1/containers.py +++ b/zun/api/controllers/v1/containers.py @@ -337,7 +337,7 @@ class ContainersController(rest.RestController): utils.validate_container_state(container, 'delete') context = pecan.request.context pecan.request.rpcapi.container_delete(context, container, force) - container.destroy() + container.destroy(context) pecan.response.status = 204 @pecan.expose('json') diff --git a/zun/common/singleton.py b/zun/common/singleton.py index 1cc593972..e4b906bbc 100644 --- a/zun/common/singleton.py +++ b/zun/common/singleton.py @@ -12,12 +12,16 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_concurrency import lockutils + class Singleton(type): _instances = {} + _semaphores = lockutils.Semaphores() def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super( - Singleton, cls).__call__(*args, **kwargs) + with lockutils.lock('singleton_lock', semaphores=cls._semaphores): + if cls not in cls._instances: + cls._instances[cls] = super( + Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] diff --git a/zun/compute/manager.py b/zun/compute/manager.py index c3b82d126..df6946a36 100644 --- a/zun/compute/manager.py +++ b/zun/compute/manager.py @@ -36,11 +36,11 @@ class Manager(object): super(Manager, self).__init__() self.driver = driver.load_container_driver(container_driver) - def _fail_container(self, container, error): + def _fail_container(self, context, container, error): container.status = fields.ContainerStatus.ERROR container.status_reason = error container.task_state = None - container.save() + container.save(context) def container_create(self, context, container): utils.spawn_n(self._do_container_create, context, container) @@ -65,7 +65,7 @@ class Manager(object): LOG.debug('Creating container: %s', container.uuid) container.task_state = fields.TaskState.SANDBOX_CREATING - container.save() + container.save(context) sandbox_id = None sandbox_image = 'kubernetes/pause' repo, tag = utils.parse_image_name(sandbox_image) @@ -77,12 +77,12 @@ class Manager(object): with excutils.save_and_reraise_exception(reraise=reraise): LOG.exception(_LE("Unexpected exception: %s"), six.text_type(e)) - self._fail_container(container, six.text_type(e)) + self._fail_container(context, container, six.text_type(e)) return self.driver.set_sandbox_id(container, sandbox_id) container.task_state = fields.TaskState.IMAGE_PULLING - container.save() + container.save(context) repo, tag = utils.parse_image_name(container.image) image_pull_policy = utils.get_image_pull_policy( container.image_pull_policy, tag) @@ -93,7 +93,7 @@ class Manager(object): with excutils.save_and_reraise_exception(reraise=reraise): LOG.error(six.text_type(e)) self._do_sandbox_cleanup(context, sandbox_id) - self._fail_container(container, six.text_type(e)) + self._fail_container(context, container, six.text_type(e)) return except exception.DockerError as e: with excutils.save_and_reraise_exception(reraise=reraise): @@ -101,24 +101,25 @@ class Manager(object): "Error occurred while calling Docker image API: %s"), six.text_type(e)) self._do_sandbox_cleanup(context, sandbox_id) - self._fail_container(container, six.text_type(e)) + self._fail_container(context, container, six.text_type(e)) return except Exception as e: with excutils.save_and_reraise_exception(reraise=reraise): LOG.exception(_LE("Unexpected exception: %s"), six.text_type(e)) self._do_sandbox_cleanup(context, sandbox_id) - self._fail_container(container, six.text_type(e)) + self._fail_container(context, container, six.text_type(e)) return container.task_state = fields.TaskState.CONTAINER_CREATING - container.save() + container.save(context) try: - container = self.driver.create(container, sandbox_id, image) + container = self.driver.create(context, container, + sandbox_id, image) container.addresses = self._get_container_addresses(context, container) container.task_state = None - container.save() + container.save(context) return container except exception.DockerError as e: with excutils.save_and_reraise_exception(reraise=reraise): @@ -126,33 +127,33 @@ class Manager(object): "Error occurred while calling Docker create API: %s"), six.text_type(e)) self._do_sandbox_cleanup(context, sandbox_id) - self._fail_container(container, six.text_type(e)) + self._fail_container(context, container, six.text_type(e)) return except Exception as e: with excutils.save_and_reraise_exception(reraise=reraise): LOG.exception(_LE("Unexpected exception: %s"), six.text_type(e)) self._do_sandbox_cleanup(context, sandbox_id) - self._fail_container(container, six.text_type(e)) + self._fail_container(context, container, six.text_type(e)) return def _do_container_start(self, context, container, reraise=False): LOG.debug('Starting container: %s', container.uuid) try: container = self.driver.start(container) - container.save() + container.save(context) return container except exception.DockerError as e: with excutils.save_and_reraise_exception(reraise=reraise): LOG.error(_LE( "Error occurred while calling Docker start API: %s"), six.text_type(e)) - self._fail_container(container, six.text_type(e)) + self._fail_container(context, container, six.text_type(e)) except Exception as e: with excutils.save_and_reraise_exception(reraise=reraise): LOG.exception(_LE("Unexpected exception: %s"), six.text_type(e)) - self._fail_container(container, six.text_type(e)) + self._fail_container(context, container, six.text_type(e)) @translate_exception def container_delete(self, context, container, force): @@ -196,7 +197,7 @@ class Manager(object): LOG.debug('Showing container: %s', container.uuid) try: container = self.driver.show(container) - container.save() + container.save(context) return container except exception.DockerError as e: LOG.error(_LE("Error occurred while calling Docker show API: %s"), @@ -210,7 +211,7 @@ class Manager(object): LOG.debug('Rebooting container: %s', container.uuid) try: container = self.driver.reboot(container, timeout) - container.save() + container.save(context) return container except exception.DockerError as e: with excutils.save_and_reraise_exception(reraise=reraise): @@ -228,7 +229,7 @@ class Manager(object): LOG.debug('Stopping container: %s', container.uuid) try: container = self.driver.stop(container, timeout) - container.save() + container.save(context) return container except exception.DockerError as e: with excutils.save_and_reraise_exception(reraise=reraise): @@ -250,7 +251,7 @@ class Manager(object): LOG.debug('Pausing container: %s', container.uuid) try: container = self.driver.pause(container) - container.save() + container.save(context) return container except exception.DockerError as e: with excutils.save_and_reraise_exception(reraise=reraise): @@ -269,7 +270,7 @@ class Manager(object): LOG.debug('Unpausing container: %s', container.uuid) try: container = self.driver.unpause(container) - container.save() + container.save(context) return container except exception.DockerError as e: with excutils.save_and_reraise_exception(reraise=reraise): @@ -315,7 +316,7 @@ class Manager(object): LOG.debug('kill signal to container: %s', container.uuid) try: container = self.driver.kill(container, signal) - container.save() + container.save(context) return container except exception.DockerError as e: with excutils.save_and_reraise_exception(reraise=reraise): diff --git a/zun/container/docker/driver.py b/zun/container/docker/driver.py index a75cff5ac..74260f741 100644 --- a/zun/container/docker/driver.py +++ b/zun/container/docker/driver.py @@ -55,7 +55,7 @@ class DockerDriver(driver.ContainerDriver): response = docker.images(repo, quiet) return response - def create(self, container, sandbox_id, image): + def create(self, context, container, sandbox_id, image): with docker_utils.docker_client() as docker: name = container.name if image['path']: @@ -91,7 +91,7 @@ class DockerDriver(driver.ContainerDriver): response = docker.create_container(image, **kwargs) container.container_id = response['Id'] container.status = fields.ContainerStatus.STOPPED - container.save() + container.save(context) return container def delete(self, container, force): diff --git a/zun/container/driver.py b/zun/container/driver.py index da5c10ee3..91f9a2713 100644 --- a/zun/container/driver.py +++ b/zun/container/driver.py @@ -59,7 +59,7 @@ def load_container_driver(container_driver=None): class ContainerDriver(object): '''Base class for container drivers.''' - def create(self, container, sandbox_name=None): + def create(self, context, container, sandbox_name=None): """Create a container.""" raise NotImplementedError() diff --git a/zun/db/etcd/api.py b/zun/db/etcd/api.py index f1af6548a..940588354 100644 --- a/zun/db/etcd/api.py +++ b/zun/db/etcd/api.py @@ -17,8 +17,10 @@ import json import etcd +from oslo_concurrency import lockutils from oslo_log import log from oslo_utils import strutils +from oslo_utils import timeutils from oslo_utils import uuidutils import six @@ -91,6 +93,7 @@ class EtcdAPI(object): def __init__(self, host, port): self.client = etcd.Client(host=host, port=port) + @lockutils.synchronized('etcd-client') def clean_all_zun_data(self): try: for d in self.client.read('/').children: @@ -123,6 +126,8 @@ class EtcdAPI(object): return resources def _process_list_result(self, res_list, limit=None, sort_key=None): + if len(res_list) == 0: + return [] sorted_res_list = res_list if sort_key: if not hasattr(res_list[0], sort_key): @@ -140,10 +145,9 @@ class EtcdAPI(object): try: res = getattr(self.client.read('/containers'), 'children', None) except etcd.EtcdKeyNotFound: - LOG.error( - _LE("Path '/containers' does not exist, seems etcd server " - "was not been initialized appropriately for Zun.")) - raise + # Before the first container been created, path '/containers' + # does not exist. + return [] except Exception as e: LOG.error( _LE("Error occurred while reading from etcd server: %s"), @@ -184,6 +188,7 @@ class EtcdAPI(object): raise exception.ContainerAlreadyExists(field='name', value=lowername) + @lockutils.synchronized('etcd_container') def create_container(self, context, container_data): # ensure defaults are present for new containers if not container_data.get('uuid'): @@ -201,23 +206,6 @@ class EtcdAPI(object): return container - def get_container_by_id(self, context, container_id): - try: - filters = self._add_tenant_filters( - context, {'id': container_id}) - containers = self.list_container(context, filters=filters) - except etcd.EtcdKeyNotFound: - raise exception.ContainerNotFound(container=container_id) - except Exception as e: - LOG.error(_LE('Error occurred while retrieving container: %s'), - six.text_type(e)) - raise - - if len(containers) == 0: - raise exception.ContainerNotFound(container=container_id) - - return containers[0] - def get_container_by_uuid(self, context, container_uuid): try: res = self.client.read('/containers/' + container_uuid) @@ -256,21 +244,13 @@ class EtcdAPI(object): return containers[0] - def _get_container_by_ident(self, context, container_ident): - if strutils.is_int_like(container_ident): - container = self.get_container_by_id(context, container_ident) - elif uuidutils.is_uuid_like(container_ident): - container = self.get_container_by_uuid(context, container_ident) - else: - raise exception.InvalidIdentity(identity=container_ident) - - return container - - def destroy_container(self, context, container_ident): - container = self._get_container_by_ident(context, container_ident) + @lockutils.synchronized('etcd_container') + def destroy_container(self, context, container_uuid): + container = self.get_container_by_uuid(context, container_uuid) self.client.delete('/containers/' + container.uuid) - def update_container(self, context, container_ident, values): + @lockutils.synchronized('etcd_container') + def update_container(self, context, container_uuid, values): # NOTE(yuywz): Update would fail if any other client # write '/containers/$CONTAINER_UUID' in the meanwhile if 'uuid' in values: @@ -281,15 +261,15 @@ class EtcdAPI(object): self._validate_unique_container_name(context, values['name']) try: - target_uuid = self._get_container_by_ident( - context, container_ident).uuid + target_uuid = self.get_container_by_uuid( + context, container_uuid).uuid target = self.client.read('/containers/' + target_uuid) target_value = json.loads(target.value) target_value.update(values) target.value = json.dumps(target_value) self.client.update(target) except etcd.EtcdKeyNotFound: - raise exception.ContainerNotFound(container=container_ident) + raise exception.ContainerNotFound(container=container_uuid) except Exception as e: LOG.error(_LE('Error occurred while updating container: %s'), six.text_type(e)) @@ -297,7 +277,9 @@ class EtcdAPI(object): return translate_etcd_result(target, 'container') + @lockutils.synchronized('etcd_zunservice') def create_zun_service(self, values): + values['created_at'] = timeutils.isotime() zun_service = models.ZunService(values) zun_service.save() return zun_service @@ -329,6 +311,7 @@ class EtcdAPI(object): def get_zun_service(self, host, binary): try: + service = None res = self.client.read('/zun_services/' + host + '_' + binary) service = translate_etcd_result(res, 'zun_service') except etcd.EtcdKeyNotFound: @@ -337,9 +320,10 @@ class EtcdAPI(object): LOG.error(_LE('Error occurred while retrieving zun service: %s'), six.text_type(e)) raise + finally: + return service - return service - + @lockutils.synchronized('etcd_zunservice') def destroy_zun_service(self, host, binary): try: self.client.delete('/zun_services/' + host + '_' + binary) @@ -350,10 +334,12 @@ class EtcdAPI(object): six.text_type(e)) raise + @lockutils.synchronized('etcd_zunservice') def update_zun_service(self, host, binary, values): try: target = self.client.read('/zun_services/' + host + '_' + binary) target_value = json.loads(target.value) + values['updated_at'] = timeutils.isotime() target_value.update(values) target.value = json.dumps(target_value) self.client.update(target) @@ -364,6 +350,7 @@ class EtcdAPI(object): six.text_type(e)) raise + @lockutils.synchronized('etcd_image') def pull_image(self, context, values): if not values.get('uuid'): values['uuid'] = uuidutils.generate_uuid() @@ -378,6 +365,7 @@ class EtcdAPI(object): image.save() return image + @lockutils.synchronized('etcd_image') def update_image(self, image_uuid, values): if 'uuid' in values: msg = _('Cannot overwrite UUID for an existing image.') @@ -403,10 +391,9 @@ class EtcdAPI(object): try: res = getattr(self.client.read('/images'), 'children', None) except etcd.EtcdKeyNotFound: - LOG.error( - _LE("Path '/images' does not exist, seems etcd server " - "was not been initialized appropriately for Zun.")) - raise + # Before the first image been pulled, path '/image' does + # not exist. + return [] except Exception as e: LOG.error( _LE("Error occurred while reading from etcd server: %s"), diff --git a/zun/db/etcd/models.py b/zun/db/etcd/models.py index 3b2368580..9889cfebf 100644 --- a/zun/db/etcd/models.py +++ b/zun/db/etcd/models.py @@ -17,6 +17,7 @@ etcd models """ import etcd +import json from zun.common import exception import zun.db.etcd as db @@ -66,14 +67,14 @@ class Base(object): if self.path_already_exist(client, path): raise exception.ResourceExists(name=getattr(self, '__class__')) - client.write(path, self.as_dict()) + client.write(path, json.dumps(self.as_dict())) return class ZunService(Base): """Represents health status of various zun services""" - _path = '/zun_service' + _path = '/zun_services' _fields = objects.ZunService.fields.keys() @@ -81,6 +82,10 @@ class ZunService(Base): self.path = ZunService.path() for f in ZunService.fields(): setattr(self, f, None) + self.id = 1 + self.disabled = False + self.forced_down = False + self.report_count = 0 self.update(service_data) @classmethod @@ -101,7 +106,7 @@ class ZunService(Base): raise exception.ZunServiceAlreadyExists(host=self.host, binary=self.binary) - client.write(path, self.as_dict()) + client.write(path, json.dumps(self.as_dict())) return @@ -116,6 +121,7 @@ class Container(Base): self.path = Container.path() for f in Container.fields(): setattr(self, f, None) + self.id = 1 self.update(container_data) @classmethod @@ -138,6 +144,7 @@ class Image(Base): self.path = Image.path() for f in Image.fields(): setattr(self, f, None) + self.id = 1 self.update(image_data) @classmethod diff --git a/zun/tests/unit/api/base.py b/zun/tests/unit/api/base.py index 5f1e21616..d46bd7a12 100644 --- a/zun/tests/unit/api/base.py +++ b/zun/tests/unit/api/base.py @@ -16,6 +16,7 @@ # NOTE(deva): import auth_token so we can override a config option from keystonemiddleware import auth_token # noqa +from oslo_config import cfg import pecan import pecan.testing from six.moves.urllib import parse as urlparse @@ -37,6 +38,10 @@ class FunctionalTest(base.DbTestCase): def setUp(self): super(FunctionalTest, self).setUp() + # NOTE(yuywz): In API test cases, we use sqllite as the DB + # backend, so we should set 'db_type' to 'sql' to access + # sqllite DB with sqlalchemy api. + cfg.CONF.set_override('db_type', 'sql') zun.conf.CONF.set_override("auth_version", "v2.0", group='keystone_authtoken', enforce_type=True) diff --git a/zun/tests/unit/api/controllers/v1/test_containers.py b/zun/tests/unit/api/controllers/v1/test_containers.py index 3022b3734..ce83ecde2 100644 --- a/zun/tests/unit/api/controllers/v1/test_containers.py +++ b/zun/tests/unit/api/controllers/v1/test_containers.py @@ -693,7 +693,7 @@ class TestContainerController(api_base.FunctionalTest): self.assertEqual(204, response.status_int) mock_container_delete.assert_called_once_with( mock.ANY, test_container_obj, False) - mock_destroy.assert_called_once_with() + mock_destroy.assert_called_once_with(mock.ANY) def test_delete_by_uuid_invalid_state(self): uuid = uuidutils.generate_uuid() @@ -729,7 +729,7 @@ class TestContainerController(api_base.FunctionalTest): self.assertEqual(204, response.status_int) mock_container_delete.assert_called_once_with( mock.ANY, test_container_obj, False) - mock_destroy.assert_called_once_with() + mock_destroy.assert_called_once_with(mock.ANY) @patch('zun.common.utils.validate_container_state') @patch('zun.compute.rpcapi.API.container_kill') diff --git a/zun/tests/unit/compute/test_compute_manager.py b/zun/tests/unit/compute/test_compute_manager.py index 364dc592c..8a86f0f6b 100644 --- a/zun/tests/unit/compute/test_compute_manager.py +++ b/zun/tests/unit/compute/test_compute_manager.py @@ -37,7 +37,8 @@ class TestManager(base.TestCase): @mock.patch.object(Container, 'save') def test_fail_container(self, mock_save): container = Container(self.context, **utils.get_test_container()) - self.compute_manager._fail_container(container, "Creation Failed") + self.compute_manager._fail_container(self.context, container, + "Creation Failed") self.assertEqual(fields.ContainerStatus.ERROR, container.status) self.assertEqual("Creation Failed", container.status_reason) self.assertIsNone(container.task_state) @@ -52,10 +53,11 @@ class TestManager(base.TestCase): mock_pull.return_value = 'fake_path' mock_create_sandbox.return_value = 'fake_id' self.compute_manager._do_container_create(self.context, container) - mock_save.assert_called_with() + mock_save.assert_called_with(self.context) mock_pull.assert_any_call(self.context, container.image, 'latest', 'always') - mock_create.assert_called_once_with(container, 'fake_id', 'fake_path') + mock_create.assert_called_once_with(self.context, container, + 'fake_id', 'fake_path') @mock.patch.object(Container, 'save') @mock.patch.object(fake_driver, 'create_sandbox') @@ -67,7 +69,8 @@ class TestManager(base.TestCase): mock_pull.side_effect = exception.DockerError("Pull Failed") mock_create_sandbox.return_value = mock.MagicMock() self.compute_manager._do_container_create(self.context, container) - mock_fail.assert_called_once_with(container, "Pull Failed") + mock_fail.assert_called_once_with(self.context, + container, "Pull Failed") @mock.patch.object(Container, 'save') @mock.patch.object(fake_driver, 'create_sandbox') @@ -79,7 +82,8 @@ class TestManager(base.TestCase): mock_pull.side_effect = exception.ImageNotFound("Image Not Found") mock_create_sandbox.return_value = mock.MagicMock() self.compute_manager._do_container_create(self.context, container) - mock_fail.assert_called_once_with(container, "Image Not Found") + mock_fail.assert_called_once_with(self.context, + container, "Image Not Found") @mock.patch.object(Container, 'save') @mock.patch.object(fake_driver, 'create_sandbox') @@ -92,7 +96,8 @@ class TestManager(base.TestCase): message="Image Not Found") mock_create_sandbox.return_value = mock.MagicMock() self.compute_manager._do_container_create(self.context, container) - mock_fail.assert_called_once_with(container, "Image Not Found") + mock_fail.assert_called_once_with(self.context, + container, "Image Not Found") @mock.patch.object(Container, 'save') @mock.patch('zun.image.driver.pull_image') @@ -107,7 +112,8 @@ class TestManager(base.TestCase): mock_create.side_effect = exception.DockerError("Creation Failed") mock_create_sandbox.return_value = mock.MagicMock() self.compute_manager._do_container_create(self.context, container) - mock_fail.assert_called_once_with(container, "Creation Failed") + mock_fail.assert_called_once_with(self.context, + container, "Creation Failed") @mock.patch.object(Container, 'save') @mock.patch('zun.image.driver.pull_image') @@ -120,10 +126,11 @@ class TestManager(base.TestCase): mock_create.return_value = container container.status = 'Stopped' self.compute_manager._do_container_run(self.context, container) - mock_save.assert_called_with() + mock_save.assert_called_with(self.context) mock_pull.assert_any_call(self.context, container.image, 'latest', 'always') - mock_create.assert_called_once_with(container, None, 'fake_path') + mock_create.assert_called_once_with(self.context, container, + None, 'fake_path') mock_start.assert_called_once_with(container) @mock.patch.object(Container, 'save') @@ -136,8 +143,9 @@ class TestManager(base.TestCase): message="Image Not Found") self.compute_manager._do_container_run(self.context, container) - mock_save.assert_called_with() - mock_fail.assert_called_with(container, 'Image Not Found') + mock_save.assert_called_with(self.context) + mock_fail.assert_called_with(self.context, + container, 'Image Not Found') mock_pull.assert_called_once_with(self.context, 'kubernetes/pause', 'latest', 'ifnotpresent') @@ -151,8 +159,9 @@ class TestManager(base.TestCase): message="Image Not Found") self.compute_manager._do_container_run(self.context, container) - mock_save.assert_called_with() - mock_fail.assert_called_with(container, 'Image Not Found') + mock_save.assert_called_with(self.context) + mock_fail.assert_called_with(self.context, + container, 'Image Not Found') mock_pull.assert_called_once_with(self.context, 'kubernetes/pause', 'latest', 'ifnotpresent') @@ -166,8 +175,9 @@ class TestManager(base.TestCase): message="Docker Error occurred") self.compute_manager._do_container_run(self.context, container) - mock_save.assert_called_with() - mock_fail.assert_called_with(container, 'Docker Error occurred') + mock_save.assert_called_with(self.context) + mock_fail.assert_called_with(self.context, + container, 'Docker Error occurred') mock_pull.assert_called_once_with(self.context, 'kubernetes/pause', 'latest', 'ifnotpresent') @@ -184,11 +194,12 @@ class TestManager(base.TestCase): message="Docker Error occurred") self.compute_manager._do_container_run(self.context, container) - mock_save.assert_called_with() - mock_fail.assert_called_with(container, 'Docker Error occurred') + mock_save.assert_called_with(self.context) + mock_fail.assert_called_with(self.context, + container, 'Docker Error occurred') mock_pull.assert_any_call(self.context, container.image, 'latest', 'always') - mock_create.assert_called_once_with(container, None, + mock_create.assert_called_once_with(self.context, container, None, {'name': 'nginx', 'path': None}) @mock.patch.object(fake_driver, 'delete') diff --git a/zun/tests/unit/container/base.py b/zun/tests/unit/container/base.py index 4fcb02ad4..11c6c1796 100644 --- a/zun/tests/unit/container/base.py +++ b/zun/tests/unit/container/base.py @@ -1,36 +1,40 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import zun.conf - -from zun.db import api as db_api -from zun.db.sqlalchemy import api as sqla_api -from zun.db.sqlalchemy import migration -from zun.tests import base -from zun.tests.unit.db.base import Database - -CONF = zun.conf.CONF - -_DB_CACHE = None - - -class DriverTestCase(base.TestCase): - def setUp(self): - super(DriverTestCase, self).setUp() - - self.dbapi = db_api.get_instance() - - global _DB_CACHE - if not _DB_CACHE: - _DB_CACHE = Database(sqla_api, migration, - sql_connection=CONF.database.connection) - self.useFixture(_DB_CACHE) +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +import zun.conf +from zun.db import api as db_api +from zun.db.sqlalchemy import api as sqla_api +from zun.db.sqlalchemy import migration +from zun.tests import base +from zun.tests.unit.db.base import Database + +CONF = zun.conf.CONF + +_DB_CACHE = None + + +class DriverTestCase(base.TestCase): + def setUp(self): + super(DriverTestCase, self).setUp() + # NOTE(yuywz): In driver test cases, we use sqllite as + # the DB backend, so we should set 'db_type' to 'sql' + # to access sqllite DB with sqlalchemy api. + cfg.CONF.set_override('db_type', 'sql') + self.dbapi = db_api.get_instance() + + global _DB_CACHE + if not _DB_CACHE: + _DB_CACHE = Database(sqla_api, migration, + sql_connection=CONF.database.connection) + self.useFixture(_DB_CACHE) diff --git a/zun/tests/unit/container/docker/test_docker_driver.py b/zun/tests/unit/container/docker/test_docker_driver.py index 9f2296d80..2bc841f27 100644 --- a/zun/tests/unit/container/docker/test_docker_driver.py +++ b/zun/tests/unit/container/docker/test_docker_driver.py @@ -19,6 +19,7 @@ from zun.container.docker import utils as docker_utils from zun.objects import fields from zun.tests.unit.container import base from zun.tests.unit.db import utils as db_utils +import zun.tests.unit.objects.utils as obj_utils class TestDockerDriver(base.DriverTestCase): @@ -53,15 +54,16 @@ class TestDockerDriver(base.DriverTestCase): self.driver.images(repo='test') self.mock_docker.images.assert_called_once_with('test', False) - def test_create_image_path_is_none(self): + @mock.patch('zun.objects.container.Container.save') + def test_create_image_path_is_none(self, mock_save): self.mock_docker.create_host_config = mock.Mock( return_value={'Id1': 'val1', 'key2': 'val2'}) self.mock_docker.create_container = mock.Mock( return_value={'Id': 'val1', 'key1': 'val2'}) db_image = {'path': ''} - db_container = db_utils.create_test_container(context=self.context) - result_container = self.driver.create(db_container, 'test_sandbox', - db_image) + container = obj_utils.get_test_container(context=self.context) + result_container = self.driver.create(self.context, container, + 'test_sandbox', db_image) host_config = {} host_config['network_mode'] = 'container:test_sandbox' @@ -82,12 +84,13 @@ class TestDockerDriver(base.DriverTestCase): 'host_config': {'Id1': 'val1', 'key2': 'val2'}, } self.mock_docker.create_container.assert_called_once_with( - db_container.image, **kwargs) + container.image, **kwargs) self.assertEqual(result_container.container_id, 'val1') self.assertEqual(result_container.status, fields.ContainerStatus.STOPPED) - def test_create_image_path_is_not_none(self): + @mock.patch('zun.objects.container.Container.save') + def test_create_image_path_is_not_none(self, mock_save): self.mock_docker.load_image = mock.Mock(return_value='load_test') self.mock_docker.create_host_config = mock.Mock( return_value={'Id1': 'val1', 'key2': 'val2'}) @@ -95,9 +98,10 @@ class TestDockerDriver(base.DriverTestCase): return_value={'Id': 'val1', 'key1': 'val2'}) mock_open_file = mock.mock_open(read_data='test_data') with mock.patch('zun.container.docker.driver.open', mock_open_file): - db_container = db_utils.create_test_container(context=self.context) - result_container = self.driver.create(db_container, 'test_sandbox', - {'path': 'test_path'}) + container = obj_utils.get_test_container(context=self.context) + result_container = self.driver.create( + self.context, container, + 'test_sandbox', {'path': 'test_path'}) self.mock_docker.load_image.assert_called_once_with('test_data') host_config = {} @@ -119,7 +123,7 @@ class TestDockerDriver(base.DriverTestCase): 'name': 'zun-ea8e2a25-2901-438d-8157-de7ffd68d051', } self.mock_docker.create_container.assert_called_once_with( - db_container.image, **kwargs) + container.image, **kwargs) self.assertEqual(result_container.container_id, 'val1') self.assertEqual(result_container.status, fields.ContainerStatus.STOPPED) diff --git a/zun/tests/unit/db/test_container.py b/zun/tests/unit/db/test_container.py index 1142ff86c..684c879fb 100644 --- a/zun/tests/unit/db/test_container.py +++ b/zun/tests/unit/db/test_container.py @@ -281,17 +281,6 @@ class EtcdDbContainerTestCase(base.DbTestCase): utils.create_test_container, context=self.context) - @mock.patch.object(etcd_client, 'read') - @mock.patch.object(etcd_client, 'write') - def test_get_container_by_id(self, mock_write, mock_read): - mock_read.side_effect = etcd.EtcdKeyNotFound - container = utils.create_test_container(context=self.context) - mock_read.side_effect = lambda *args: FakeEtcdMutlipleResult( - [container.as_dict()]) - res = dbapi.Connection.get_container_by_id(self.context, container.id) - self.assertEqual(container.id, res.id) - self.assertEqual(container.uuid, res.uuid) - @mock.patch.object(etcd_client, 'read') @mock.patch.object(etcd_client, 'write') def test_get_container_by_uuid(self, mock_write, mock_read): @@ -319,9 +308,6 @@ class EtcdDbContainerTestCase(base.DbTestCase): @mock.patch.object(etcd_client, 'read') def test_get_container_that_does_not_exist(self, mock_read): mock_read.side_effect = etcd.EtcdKeyNotFound - self.assertRaises(exception.ContainerNotFound, - dbapi.Connection.get_container_by_id, - self.context, 99) self.assertRaises(exception.ContainerNotFound, dbapi.Connection.get_container_by_uuid, self.context, @@ -410,9 +396,9 @@ class EtcdDbContainerTestCase(base.DbTestCase): def test_destroy_container(self, mock_delete, mock_write, mock_read): mock_read.side_effect = etcd.EtcdKeyNotFound container = utils.create_test_container(context=self.context) - mock_read.side_effect = lambda *args: FakeEtcdMutlipleResult( - [container.as_dict()]) - dbapi.Connection.destroy_container(self.context, container.id) + mock_read.side_effect = lambda *args: FakeEtcdResult( + container.as_dict()) + dbapi.Connection.destroy_container(self.context, container.uuid) mock_delete.assert_called_once_with('/containers/%s' % container.uuid) @mock.patch.object(etcd_client, 'read') diff --git a/zun/tests/unit/db/test_image.py b/zun/tests/unit/db/test_image.py index d58822bdc..8e776a46a 100644 --- a/zun/tests/unit/db/test_image.py +++ b/zun/tests/unit/db/test_image.py @@ -29,6 +29,10 @@ from zun.tests.unit.db.utils import FakeEtcdResult class DbImageTestCase(base.DbTestCase): + def setUp(self): + cfg.CONF.set_override('db_type', 'sql') + super(DbImageTestCase, self).setUp() + def test_pull_image(self): utils.create_test_image(context=self.context, repo="ubuntu:latest") diff --git a/zun/tests/unit/db/test_zun_service.py b/zun/tests/unit/db/test_zun_service.py index 8388d8ed4..1015de1ac 100644 --- a/zun/tests/unit/db/test_zun_service.py +++ b/zun/tests/unit/db/test_zun_service.py @@ -65,9 +65,9 @@ class EtcdDbZunServiceTestCase(base.DbTestCase): def test_get_zun_service_not_found(self, mock_write, mock_read): mock_read.side_effect = etcd.EtcdKeyNotFound zun_service = utils.create_test_zun_service() - self.assertRaises(exception.ZunServiceNotFound, - dbapi.Connection.get_zun_service, self.context, - 'wrong_host_name', zun_service.binary) + res = dbapi.Connection.get_zun_service( + self.context, 'wrong_host_name', zun_service.binary) + self.assertIsNone(res) @mock.patch.object(etcd_client, 'read') @mock.patch.object(etcd_client, 'write') diff --git a/zun/tests/unit/objects/test_container.py b/zun/tests/unit/objects/test_container.py index 7b19e74ff..f543c26dc 100644 --- a/zun/tests/unit/objects/test_container.py +++ b/zun/tests/unit/objects/test_container.py @@ -14,6 +14,7 @@ # under the License. import mock + from oslo_utils import uuidutils from testtools.matchers import HasLength @@ -83,13 +84,16 @@ class TestContainerObject(base.DbTestCase): self.assertEqual(self.context, container._context) def test_status_reason_in_fields(self): - container = objects.Container(self.context, **self.fake_container) - self.assertTrue(hasattr(container, 'status_reason')) - container.status_reason = "Docker Error happened" - container.create(self.context) - containers = objects.Container.list(self.context) - self.assertTrue(hasattr(containers[0], 'status_reason')) - self.assertEqual("Docker Error happened", containers[0].status_reason) + with mock.patch.object(self.dbapi, 'create_container', + autospec=True) as mock_create_container: + mock_create_container.return_value = self.fake_container + container = objects.Container(self.context, **self.fake_container) + self.assertTrue(hasattr(container, 'status_reason')) + container.status_reason = "Docker Error happened" + container.create(self.context) + self.assertEqual( + "Docker Error happened", + mock_create_container.call_args_list[0][0][1]['status_reason']) def test_destroy(self): uuid = self.fake_container['uuid']