diff --git a/etc/zun/policy.json b/etc/zun/policy.json index 0ed01bacf..8c9785657 100644 --- a/etc/zun/policy.json +++ b/etc/zun/policy.json @@ -27,7 +27,7 @@ "container:get_archive": "rule:admin_or_user", "container:put_archive": "rule:admin_or_user", "container:stats": "rule:admin_or_user", - + "container:commit": "rule:admin_or_user", "image:pull": "rule:default", "image:get_all": "rule:default", "image:search": "rule:default", diff --git a/zun/api/controllers/v1/containers.py b/zun/api/controllers/v1/containers.py index 01c4a2740..58f4e6ec3 100644 --- a/zun/api/controllers/v1/containers.py +++ b/zun/api/controllers/v1/containers.py @@ -94,6 +94,7 @@ class ContainersController(base.Controller): 'get_archive': ['GET'], 'put_archive': ['POST'], 'stats': ['GET'], + 'commit': ['POST'], } @pecan.expose('json') @@ -543,3 +544,18 @@ class ContainersController(base.Controller): context = pecan.request.context compute_api = pecan.request.compute_api return compute_api.container_stats(context, container) + + @pecan.expose('json') + @exception.wrap_pecan_controller_exception + @validation.validate_query_param(pecan.request, schema.query_param_commit) + def commit(self, container_id, **kw): + container = _get_container(container_id) + check_policy_on_container(container.as_dict(), "container:commit") + utils.validate_container_state(container, 'commit') + LOG.debug('Calling compute.container_commit %s ' % (container.uuid)) + context = pecan.request.context + compute_api = pecan.request.compute_api + compute_api.container_commit(context, container, + kw.get('repository', None), + kw.get('tag', None)) + pecan.response.status = 202 diff --git a/zun/api/controllers/v1/schemas/containers.py b/zun/api/controllers/v1/schemas/containers.py index ca1860a2b..f48b82fa7 100644 --- a/zun/api/controllers/v1/schemas/containers.py +++ b/zun/api/controllers/v1/schemas/containers.py @@ -136,3 +136,13 @@ query_param_execute_command = { }, 'additionalProperties': False } + +query_param_commit = { + 'type': 'object', + 'properties': { + 'repository': parameter_types.string_ps_args, + 'tag': parameter_types.string_ps_args + }, + 'required': ['repository'], + 'additionalProperties': False +} diff --git a/zun/common/utils.py b/zun/common/utils.py index a368d4e4c..1c584135e 100644 --- a/zun/common/utils.py +++ b/zun/common/utils.py @@ -36,6 +36,7 @@ LOG = logging.getLogger(__name__) VALID_STATES = { + 'commit': [consts.RUNNING, consts.STOPPED, consts.PAUSED], 'delete': [consts.CREATED, consts.ERROR, consts.STOPPED], 'delete_force': [consts.CREATED, consts.CREATING, consts.ERROR, consts.RUNNING, consts.STOPPED, consts.UNKNOWN], diff --git a/zun/compute/api.py b/zun/compute/api.py index 64153d4ef..87c56fba9 100644 --- a/zun/compute/api.py +++ b/zun/compute/api.py @@ -112,6 +112,9 @@ class API(object): def container_stats(self, context, container): return self.rpcapi.container_stats(context, container) + def container_commit(self, context, container, *args): + return self.rpcapi.container_commit(context, container, *args) + def image_pull(self, context, image, *args): return self.rpcapi.image_pull(context, image, *args) diff --git a/zun/compute/manager.py b/zun/compute/manager.py index d21c498be..c1cc68457 100755 --- a/zun/compute/manager.py +++ b/zun/compute/manager.py @@ -24,6 +24,7 @@ from zun.common.utils import translate_exception import zun.conf from zun.container import driver from zun.image import driver as image_driver +from zun.image.glance import driver as glance CONF = zun.conf.CONF LOG = logging.getLogger(__name__) @@ -475,6 +476,41 @@ class Manager(object): LOG.exception("Unexpected exception: %s", six.text_type(e)) raise + @translate_exception + def container_commit(self, context, container, repository, tag=None): + LOG.debug('Commit the container: %s', container.uuid) + utils.spawn_n(self._do_container_commit, context, container, + repository, tag) + + def _do_container_image_upload(self, context, data, repo, tag): + try: + image_driver.upload_image(context, repo, + tag, data, + glance.GlanceDriver()) + except Exception as e: + LOG.exception("Unexpected exception while uploading image: %s", + six.text_type(e)) + raise + + def _do_container_commit(self, context, container, repository, tag=None): + LOG.debug('Creating image...') + container_image = None + container_image_id = None + if tag is None: + tag = 'latest' + + try: + container_image_id = self.driver.commit(container, + repository, tag) + container_image = self.driver.get_image(repository + ':' + tag) + except exception.DockerError as e: + LOG.error("Error occurred while calling docker commit API: %s", + six.text_type(e)) + raise + LOG.debug('Upload image %s to glance' % container_image_id) + self._do_container_image_upload(context, container_image, + repository, tag) + def image_pull(self, context, image): utils.spawn_n(self._do_image_pull, context, image) diff --git a/zun/compute/rpcapi.py b/zun/compute/rpcapi.py index fcd2f3fc4..c0d679b1d 100644 --- a/zun/compute/rpcapi.py +++ b/zun/compute/rpcapi.py @@ -146,6 +146,11 @@ class API(rpc_service.API): return self._call(container.host, 'container_stats', container=container) + @check_container_host + def container_commit(self, context, container, repository, tag): + return self._cast(container.host, 'container_commit', + container=container, repository=repository, tag=tag) + def image_pull(self, context, image): # NOTE(hongbin): Image API doesn't support multiple compute nodes # scenario yet, so we temporarily set host to None and rpc will diff --git a/zun/container/docker/driver.py b/zun/container/docker/driver.py index eb7aa1e03..cf658665f 100644 --- a/zun/container/docker/driver.py +++ b/zun/container/docker/driver.py @@ -58,6 +58,12 @@ class DockerDriver(driver.ContainerDriver): image_dict = docker.inspect_image(image) return image_dict + def get_image(self, name): + LOG.debug('Obtaining image %s' % name) + with docker_utils.docker_client() as docker: + response = docker.get_image(name) + return response + def images(self, repo, quiet=False): with docker_utils.docker_client() as docker: response = docker.images(repo, quiet) @@ -469,6 +475,19 @@ class DockerDriver(driver.ContainerDriver): return docker.stats(container.container_id, decode=False, stream=False) + @check_container_id + def commit(self, container, repository=None, tag=None): + with docker_utils.docker_client() as docker: + repository = str(repository) + try: + if tag is None or tag == "None": + return docker.commit(container.container_id, repository) + else: + return docker.commit(container.container_id, + repository, tag) + except errors.APIError: + raise + def _encode_utf8(self, value): if six.PY2 and not isinstance(value, unicode): value = unicode(value) diff --git a/zun/container/driver.py b/zun/container/driver.py index 23530860a..5935ae312 100644 --- a/zun/container/driver.py +++ b/zun/container/driver.py @@ -64,6 +64,10 @@ class ContainerDriver(object): """Create a container.""" raise NotImplementedError() + def commit(self, container, repository, tag): + """commit a container.""" + raise NotImplementedError() + def delete(self, container, force): """Delete a container.""" raise NotImplementedError() diff --git a/zun/image/driver.py b/zun/image/driver.py index 8ec753b7b..4ecdc6f8d 100644 --- a/zun/image/driver.py +++ b/zun/image/driver.py @@ -104,6 +104,27 @@ def search_image(context, image_name, image_driver, exact_match): return images +def upload_image(context, image_name, image_tag, image_data, + image_driver): + img = None + try: + img = image_driver.create_image(context, image_name) + img = image_driver.update_image(context, + img.id, + tag=image_tag) + # Image data has to match the image format. + # contain format defaults to 'docker'; + # disk format defaults to 'qcow2'. + img = image_driver.upload_image_data(context, + img.id, + image_data) + except Exception as e: + LOG.exception('Unknown exception occurred while uploading image: %s', + six.text_type(e)) + raise exception.ZunException(six.text_type(e)) + return img + + class ContainerImageDriver(object): '''Base class for container image driver.''' @@ -114,3 +135,16 @@ class ContainerImageDriver(object): def search_image(self, context, repo, tag): """Search an image.""" raise NotImplementedError() + + def create_image(self, context, image_name): + """Create an image.""" + raise NotImplementedError() + + def update_image(self, context, img_id, container_fmt=None, + disk_fmt=None, tag=None): + """Update an image.""" + raise NotImplementedError() + + def upload_image_data(self, context, img_id, data): + """Upload an image.""" + raise NotImplementedError() diff --git a/zun/image/glance/driver.py b/zun/image/glance/driver.py index 8995a4980..104f6f949 100644 --- a/zun/image/glance/driver.py +++ b/zun/image/glance/driver.py @@ -123,3 +123,37 @@ class GlanceDriver(driver.ContainerImageDriver): return images except Exception as e: raise exception.ZunException(six.text_type(e)) + + def create_image(self, context, image_name): + """Create an image.""" + LOG.debug('Creating a new image in glance %s' % image_name) + try: + img = utils.create_image(context, image_name) + return img + except Exception as e: + raise exception.ZunException(six.text_type(e)) + + def update_image(self, context, img_id, disk_format='qcow2', + container_format='docker', tag=None): + """Update an image.""" + LOG.debug('Updating an image %s in glance' % img_id) + try: + if tag is not None: + tags = [] + tags.append(tag) + img = utils.update_image_tags(context, img_id, + tags) + img = utils.update_image_format(context, img_id, disk_format, + container_format) + return img + except Exception as e: + raise exception.ZunException(six.text_type(e)) + + def upload_image_data(self, context, img_id, data): + """Update an image.""" + LOG.debug('Uploading an image to glance %s' % img_id) + try: + img = utils.upload_image_data(context, img_id, data) + return img + except Exception as e: + raise exception.ZunException(six.text_type(e)) diff --git a/zun/image/glance/utils.py b/zun/image/glance/utils.py index d0a9719fa..5846fdb7d 100644 --- a/zun/image/glance/utils.py +++ b/zun/image/glance/utils.py @@ -19,6 +19,9 @@ from oslo_utils import uuidutils from zun.common import clients from zun.common import exception +from oslo_log import log as logging +LOG = logging.getLogger(__name__) + def create_glanceclient(context): """Creates glance client object. @@ -32,6 +35,7 @@ def create_glanceclient(context): def find_image(context, image_ident): matches = find_images(context, image_ident, exact_match=True) + LOG.debug('Found matches %s ' % matches) if len(matches) == 0: raise exception.ImageNotFound(image=image_ident) if len(matches) > 1: @@ -62,3 +66,34 @@ def find_images(context, image_ident, exact_match): images = [i for i in images if image_ident in i.name] return images + + +def create_image(context, image_name): + """Create an image.""" + glance = create_glanceclient(context) + img = glance.images.create(name=image_name) + return img + + +def update_image_format(context, img_id, disk_format, + container_format): + """Update container format of an image.""" + glance = create_glanceclient(context) + img = glance.images.update(img_id, disk_format=disk_format, + container_format=container_format) + return img + + +def update_image_tags(context, img_id, tags): + """Adding new tags to the tag list of an image.""" + glance = create_glanceclient(context) + img = glance.images.update(img_id, tags=tags) + return img + + +def upload_image_data(context, img_id, data): + """Upload an image.""" + LOG.debug('Upload image %s ' % img_id) + glance = create_glanceclient(context) + img = glance.images.upload(img_id, data) + return img diff --git a/zun/tests/unit/api/controllers/v1/test_containers.py b/zun/tests/unit/api/controllers/v1/test_containers.py index fa3354bbd..9b3e31ca7 100644 --- a/zun/tests/unit/api/controllers/v1/test_containers.py +++ b/zun/tests/unit/api/controllers/v1/test_containers.py @@ -1231,6 +1231,55 @@ class TestContainerController(api_base.FunctionalTest): self.assertEqual(200, response.status_int) self.assertTrue(mock_container_stats.called) + @patch('zun.common.utils.validate_container_state') + @patch('zun.compute.api.API.container_commit') + @patch('zun.objects.Container.get_by_name') + def test_commit_by_name(self, mock_get_by_name, + mock_container_commit, mock_validate): + + test_container_obj = objects.Container(self.context, + **utils.get_test_container()) + test_container = utils.get_test_container() + test_container_obj = objects.Container(self.context, **test_container) + mock_get_by_name.return_value = test_container_obj + mock_container_commit.return_value = None + container_name = test_container.get('name') + url = '/v1/containers/%s/%s/' % (container_name, 'commit') + cmd = {'repository': 'repo', 'tag': 'tag'} + response = self.app.post(url, cmd) + self.assertEqual(202, response.status_int) + mock_container_commit.assert_called_once_with( + mock.ANY, test_container_obj, cmd['repository'], cmd['tag']) + + @patch('zun.common.utils.validate_container_state') + @patch('zun.compute.api.API.container_commit') + @patch('zun.objects.Container.get_by_uuid') + def test_commit_by_uuid(self, mock_get_by_uuid, + mock_container_commit, mock_validate): + + test_container_obj = objects.Container(self.context, + **utils.get_test_container()) + test_container = utils.get_test_container() + test_container_obj = objects.Container(self.context, **test_container) + mock_get_by_uuid.return_value = test_container_obj + mock_container_commit.return_value = None + container_uuid = test_container.get('uuid') + url = '/v1/containers/%s/%s/' % (container_uuid, 'commit') + cmd = {'repository': 'repo', 'tag': 'tag'} + response = self.app.post(url, cmd) + self.assertEqual(202, response.status_int) + mock_container_commit.assert_called_once_with( + mock.ANY, test_container_obj, cmd['repository'], cmd['tag']) + + def test_commit_by_uuid_invalid_state(self): + uuid = uuidutils.generate_uuid() + cmd = {'repository': 'repo', 'tag': 'tag'} + utils.create_test_container(context=self.context, + uuid=uuid, status='Error') + with self.assertRaisesRegexp( + AppError, "Cannot commit container %s in Error state" % uuid): + self.app.post('/v1/containers/%s/commit/' % uuid, cmd) + class TestContainerEnforcement(api_base.FunctionalTest): diff --git a/zun/tests/unit/compute/test_compute_manager.py b/zun/tests/unit/compute/test_compute_manager.py index 1b6794590..9954da20e 100755 --- a/zun/tests/unit/compute/test_compute_manager.py +++ b/zun/tests/unit/compute/test_compute_manager.py @@ -14,7 +14,7 @@ import mock - +from io import StringIO from zun.common import consts from zun.common import exception from zun.compute import manager @@ -525,3 +525,26 @@ class TestManager(base.TestCase): self.assertRaises(exception.DockerError, self.compute_manager.container_exec_resize, self.context, 'fake_exec_id', "100", "100") + + @mock.patch('zun.image.driver.upload_image') + @mock.patch.object(fake_driver, 'get_image') + @mock.patch.object(fake_driver, 'commit') + def test_container_commit(self, mock_commit, + mock_get_image, mock_upload_image): + container = Container(self.context, **utils.get_test_container()) + mock_get_image_response = mock.MagicMock() + mock_get_image_response.data = StringIO().read() + mock_get_image.return_value = mock_get_image_response + mock_upload_image.return_value = mock.MagicMock() + + self.compute_manager._do_container_commit(self.context, + container, 'repo', 'tag') + mock_commit.assert_called_once_with(container, 'repo', 'tag') + + @mock.patch.object(fake_driver, 'commit') + def test_container_commit_failed(self, mock_commit): + container = Container(self.context, **utils.get_test_container()) + mock_commit.side_effect = exception.DockerError + self.assertRaises(exception.DockerError, + self.compute_manager._do_container_commit, + self.context, container, 'repo', 'tag') diff --git a/zun/tests/unit/container/docker/test_docker_driver.py b/zun/tests/unit/container/docker/test_docker_driver.py index fa79db953..47891e117 100644 --- a/zun/tests/unit/container/docker/test_docker_driver.py +++ b/zun/tests/unit/container/docker/test_docker_driver.py @@ -61,6 +61,11 @@ class TestDockerDriver(base.DriverTestCase): self.driver.inspect_image(mock_image) self.mock_docker.inspect_image.assert_called_once_with(mock_image) + def test_get_image(self): + self.mock_docker.get_image = mock.Mock() + self.driver.get_image(name='image_name') + self.mock_docker.get_image.assert_called_once_with('image_name') + def test_load_image(self): self.mock_docker.load_image = mock.Mock() mock_open_file = mock.mock_open() @@ -336,6 +341,13 @@ class TestDockerDriver(base.DriverTestCase): self.mock_docker.resize.assert_called_once_with( mock_container.container_id, 100, 100) + def test_commit(self): + self.mock_docker.commit = mock.Mock() + mock_container = mock.MagicMock() + self.driver.commit(mock_container, "repo", "tag") + self.mock_docker.commit.assert_called_once_with( + mock_container.container_id, "repo", "tag") + @mock.patch('zun.network.kuryr_network.KuryrNetwork' '.connect_container_to_network') @mock.patch('zun.container.docker.driver.DockerDriver.get_sandbox_name') diff --git a/zun/tests/unit/container/fake_driver.py b/zun/tests/unit/container/fake_driver.py index 1aefdec9a..6733ee48c 100644 --- a/zun/tests/unit/container/fake_driver.py +++ b/zun/tests/unit/container/fake_driver.py @@ -28,6 +28,9 @@ class FakeDriver(driver.ContainerDriver): def inspect_image(self, image): pass + def get_image(self, name): + pass + def images(self, repo, **kwargs): pass @@ -101,3 +104,7 @@ class FakeDriver(driver.ContainerDriver): @check_container_id def update(self, container): pass + + @check_container_id + def commit(self, container, repository, tag): + pass diff --git a/zun/tests/unit/image/glance/test_driver.py b/zun/tests/unit/image/glance/test_driver.py index a4b0c1ed9..8f794e908 100644 --- a/zun/tests/unit/image/glance/test_driver.py +++ b/zun/tests/unit/image/glance/test_driver.py @@ -135,3 +135,26 @@ class TestDriver(base.BaseTestCase): self.assertRaises(exception.ZunException, self.driver.search_image, None, 'image', None, False) self.assertTrue(mock_find_images.called) + + @mock.patch('zun.image.glance.utils.create_image') + def test_create_image(self, mock_create_image): + image_meta = mock.MagicMock() + image_meta.id = '1234' + mock_create_image.return_value = [image_meta] + ret = self.driver.create_image(None, 'image') + self.assertEqual(1, len(ret)) + self.assertTrue(mock_create_image.called) + + @mock.patch.object(driver.GlanceDriver, 'update_image') + @mock.patch('zun.image.glance.utils.update_image_tags') + @mock.patch('zun.image.glance.utils.update_image_format') + def test_update_image(self, mock_update_image_format, + mock_update_image_tags, mock_update_image): + image_meta = mock.MagicMock() + image_meta.id = '1234' + mock_update_image_tags.return_value = [image_meta] + mock_update_image_format.return_value = [image_meta] + mock_update_image.return_value = [image_meta] + ret = self.driver.update_image(None, 'id', container_format='docker') + self.assertEqual(1, len(ret)) + self.assertTrue(mock_update_image.called)