Merge "Implement container snapshot"

etc/zun/policy.json

@@ -27,7 +27,7 @@
     "container:get_archive": "rule:admin_or_user",
     "container:put_archive": "rule:admin_or_user",
     "container:stats": "rule:admin_or_user",
-
+    "container:commit": "rule:admin_or_user",
     "image:pull": "rule:default",
     "image:get_all": "rule:default",
     "image:search": "rule:default",

zun/api/controllers/v1/containers.py

@@ -94,6 +94,7 @@ class ContainersController(base.Controller):
         'get_archive': ['GET'],
         'put_archive': ['POST'],
         'stats': ['GET'],
+        'commit': ['POST'],
     }
 
     @pecan.expose('json')
@@ -543,3 +544,18 @@ class ContainersController(base.Controller):
         context = pecan.request.context
         compute_api = pecan.request.compute_api
         return compute_api.container_stats(context, container)
+
+    @pecan.expose('json')
+    @exception.wrap_pecan_controller_exception
+    @validation.validate_query_param(pecan.request, schema.query_param_commit)
+    def commit(self, container_id, **kw):
+        container = _get_container(container_id)
+        check_policy_on_container(container.as_dict(), "container:commit")
+        utils.validate_container_state(container, 'commit')
+        LOG.debug('Calling compute.container_commit %s ' % (container.uuid))
+        context = pecan.request.context
+        compute_api = pecan.request.compute_api
+        compute_api.container_commit(context, container,
+                                     kw.get('repository', None),
+                                     kw.get('tag', None))
+        pecan.response.status = 202

zun/api/controllers/v1/schemas/containers.py

@@ -136,3 +136,13 @@ query_param_execute_command = {
     },
     'additionalProperties': False
 }
+
+query_param_commit = {
+    'type': 'object',
+    'properties': {
+        'repository': parameter_types.string_ps_args,
+        'tag': parameter_types.string_ps_args
+    },
+    'required': ['repository'],
+    'additionalProperties': False
+}

zun/common/utils.py

@@ -36,6 +36,7 @@ LOG = logging.getLogger(__name__)
 
 
 VALID_STATES = {
+    'commit': [consts.RUNNING, consts.STOPPED, consts.PAUSED],
     'delete': [consts.CREATED, consts.ERROR, consts.STOPPED],
     'delete_force': [consts.CREATED, consts.CREATING, consts.ERROR,
                      consts.RUNNING, consts.STOPPED, consts.UNKNOWN],

zun/compute/api.py

@@ -112,6 +112,9 @@ class API(object):
     def container_stats(self, context, container):
         return self.rpcapi.container_stats(context, container)
 
+    def container_commit(self, context, container, *args):
+        return self.rpcapi.container_commit(context, container, *args)
+
     def image_pull(self, context, image, *args):
         return self.rpcapi.image_pull(context, image, *args)
 

zun/compute/manager.py

@@ -25,6 +25,7 @@ from zun.common.utils import translate_exception
 import zun.conf
 from zun.container import driver
 from zun.image import driver as image_driver
+from zun.image.glance import driver as glance
 
 CONF = zun.conf.CONF
 LOG = logging.getLogger(__name__)
@@ -482,6 +483,41 @@ class Manager(object):
             LOG.exception("Unexpected exception: %s", six.text_type(e))
             raise
 
+    @translate_exception
+    def container_commit(self, context, container, repository, tag=None):
+        LOG.debug('Commit the container: %s', container.uuid)
+        utils.spawn_n(self._do_container_commit, context,  container,
+                      repository, tag)
+
+    def _do_container_image_upload(self, context, data, repo, tag):
+        try:
+            image_driver.upload_image(context, repo,
+                                      tag, data,
+                                      glance.GlanceDriver())
+        except Exception as e:
+            LOG.exception("Unexpected exception while uploading image: %s",
+                          six.text_type(e))
+            raise
+
+    def _do_container_commit(self, context, container, repository, tag=None):
+        LOG.debug('Creating image...')
+        container_image = None
+        container_image_id = None
+        if tag is None:
+            tag = 'latest'
+
+        try:
+            container_image_id = self.driver.commit(container,
+                                                    repository, tag)
+            container_image = self.driver.get_image(repository + ':' + tag)
+        except exception.DockerError as e:
+            LOG.error("Error occurred while calling docker commit API: %s",
+                      six.text_type(e))
+            raise
+        LOG.debug('Upload image %s to glance' % container_image_id)
+        self._do_container_image_upload(context, container_image,
+                                        repository, tag)
+
     def image_pull(self, context, image):
         utils.spawn_n(self._do_image_pull, context, image)
 

zun/compute/rpcapi.py

@@ -146,6 +146,11 @@ class API(rpc_service.API):
         return self._call(container.host, 'container_stats',
                           container=container)
 
+    @check_container_host
+    def container_commit(self, context, container, repository, tag):
+        return self._cast(container.host, 'container_commit',
+                          container=container, repository=repository, tag=tag)
+
     def image_pull(self, context, image):
         # NOTE(hongbin): Image API doesn't support multiple compute nodes
         # scenario yet, so we temporarily set host to None and rpc will

zun/container/docker/driver.py

@@ -58,6 +58,12 @@ class DockerDriver(driver.ContainerDriver):
             image_dict = docker.inspect_image(image)
             return image_dict
 
+    def get_image(self, name):
+        LOG.debug('Obtaining image %s' % name)
+        with docker_utils.docker_client() as docker:
+            response = docker.get_image(name)
+            return response
+
     def images(self, repo, quiet=False):
         with docker_utils.docker_client() as docker:
             response = docker.images(repo, quiet)
@@ -469,6 +475,19 @@ class DockerDriver(driver.ContainerDriver):
             return docker.stats(container.container_id, decode=False,
                                 stream=False)
 
+    @check_container_id
+    def commit(self, container, repository=None, tag=None):
+        with docker_utils.docker_client() as docker:
+            repository = str(repository)
+            try:
+                if tag is None or tag == "None":
+                    return docker.commit(container.container_id, repository)
+                else:
+                    return docker.commit(container.container_id,
+                                         repository, tag)
+            except errors.APIError:
+                raise
+
     def _encode_utf8(self, value):
         if six.PY2 and not isinstance(value, unicode):
             value = unicode(value)

zun/container/driver.py

@@ -64,6 +64,10 @@ class ContainerDriver(object):
         """Create a container."""
         raise NotImplementedError()
 
+    def commit(self, container, repository, tag):
+        """commit a container."""
+        raise NotImplementedError()
+
     def delete(self, container, force):
         """Delete a container."""
         raise NotImplementedError()

zun/image/driver.py

@@ -104,6 +104,27 @@ def search_image(context, image_name, image_driver, exact_match):
     return images
 
 
+def upload_image(context, image_name, image_tag, image_data,
+                 image_driver):
+    img = None
+    try:
+        img = image_driver.create_image(context, image_name)
+        img = image_driver.update_image(context,
+                                        img.id,
+                                        tag=image_tag)
+        # Image data has to match the image format.
+        # contain format defaults to 'docker';
+        # disk format defaults to 'qcow2'.
+        img = image_driver.upload_image_data(context,
+                                             img.id,
+                                             image_data)
+    except Exception as e:
+        LOG.exception('Unknown exception occurred while uploading image: %s',
+                      six.text_type(e))
+        raise exception.ZunException(six.text_type(e))
+    return img
+
+
 class ContainerImageDriver(object):
     '''Base class for container image driver.'''
 
@@ -114,3 +135,16 @@ class ContainerImageDriver(object):
     def search_image(self, context, repo, tag):
         """Search an image."""
         raise NotImplementedError()
+
+    def create_image(self, context, image_name):
+        """Create an image."""
+        raise NotImplementedError()
+
+    def update_image(self, context, img_id, container_fmt=None,
+                     disk_fmt=None, tag=None):
+        """Update an image."""
+        raise NotImplementedError()
+
+    def upload_image_data(self, context, img_id, data):
+        """Upload an image."""
+        raise NotImplementedError()

zun/image/glance/driver.py

@@ -123,3 +123,37 @@ class GlanceDriver(driver.ContainerImageDriver):
             return images
         except Exception as e:
             raise exception.ZunException(six.text_type(e))
+
+    def create_image(self, context, image_name):
+        """Create an image."""
+        LOG.debug('Creating a new image in glance %s' % image_name)
+        try:
+            img = utils.create_image(context, image_name)
+            return img
+        except Exception as e:
+            raise exception.ZunException(six.text_type(e))
+
+    def update_image(self, context, img_id, disk_format='qcow2',
+                     container_format='docker', tag=None):
+        """Update an image."""
+        LOG.debug('Updating an image %s in glance' % img_id)
+        try:
+            if tag is not None:
+                tags = []
+                tags.append(tag)
+                img = utils.update_image_tags(context, img_id,
+                                              tags)
+            img = utils.update_image_format(context, img_id, disk_format,
+                                            container_format)
+            return img
+        except Exception as e:
+            raise exception.ZunException(six.text_type(e))
+
+    def upload_image_data(self, context, img_id, data):
+        """Update an image."""
+        LOG.debug('Uploading an image to glance %s' % img_id)
+        try:
+            img = utils.upload_image_data(context, img_id, data)
+            return img
+        except Exception as e:
+            raise exception.ZunException(six.text_type(e))

zun/image/glance/utils.py

@@ -19,6 +19,9 @@ from oslo_utils import uuidutils
 from zun.common import clients
 from zun.common import exception
 
+from oslo_log import log as logging
+LOG = logging.getLogger(__name__)
+
 
 def create_glanceclient(context):
     """Creates glance client object.
@@ -32,6 +35,7 @@ def create_glanceclient(context):
 
 def find_image(context, image_ident):
     matches = find_images(context, image_ident, exact_match=True)
+    LOG.debug('Found matches %s ' % matches)
     if len(matches) == 0:
         raise exception.ImageNotFound(image=image_ident)
     if len(matches) > 1:
@@ -62,3 +66,34 @@ def find_images(context, image_ident, exact_match):
             images = [i for i in images if image_ident in i.name]
 
     return images
+
+
+def create_image(context, image_name):
+    """Create an image."""
+    glance = create_glanceclient(context)
+    img = glance.images.create(name=image_name)
+    return img
+
+
+def update_image_format(context, img_id, disk_format,
+                        container_format):
+    """Update container format of an image."""
+    glance = create_glanceclient(context)
+    img = glance.images.update(img_id, disk_format=disk_format,
+                               container_format=container_format)
+    return img
+
+
+def update_image_tags(context, img_id, tags):
+    """Adding new tags to the tag list of an image."""
+    glance = create_glanceclient(context)
+    img = glance.images.update(img_id, tags=tags)
+    return img
+
+
+def upload_image_data(context, img_id, data):
+    """Upload an image."""
+    LOG.debug('Upload image %s ' % img_id)
+    glance = create_glanceclient(context)
+    img = glance.images.upload(img_id, data)
+    return img

zun/tests/unit/api/controllers/v1/test_containers.py

@@ -1231,6 +1231,55 @@ class TestContainerController(api_base.FunctionalTest):
         self.assertEqual(200, response.status_int)
         self.assertTrue(mock_container_stats.called)
 
+    @patch('zun.common.utils.validate_container_state')
+    @patch('zun.compute.api.API.container_commit')
+    @patch('zun.objects.Container.get_by_name')
+    def test_commit_by_name(self, mock_get_by_name,
+                            mock_container_commit, mock_validate):
+
+        test_container_obj = objects.Container(self.context,
+                                               **utils.get_test_container())
+        test_container = utils.get_test_container()
+        test_container_obj = objects.Container(self.context, **test_container)
+        mock_get_by_name.return_value = test_container_obj
+        mock_container_commit.return_value = None
+        container_name = test_container.get('name')
+        url = '/v1/containers/%s/%s/' % (container_name, 'commit')
+        cmd = {'repository': 'repo', 'tag': 'tag'}
+        response = self.app.post(url, cmd)
+        self.assertEqual(202, response.status_int)
+        mock_container_commit.assert_called_once_with(
+            mock.ANY, test_container_obj, cmd['repository'], cmd['tag'])
+
+    @patch('zun.common.utils.validate_container_state')
+    @patch('zun.compute.api.API.container_commit')
+    @patch('zun.objects.Container.get_by_uuid')
+    def test_commit_by_uuid(self, mock_get_by_uuid,
+                            mock_container_commit, mock_validate):
+
+        test_container_obj = objects.Container(self.context,
+                                               **utils.get_test_container())
+        test_container = utils.get_test_container()
+        test_container_obj = objects.Container(self.context, **test_container)
+        mock_get_by_uuid.return_value = test_container_obj
+        mock_container_commit.return_value = None
+        container_uuid = test_container.get('uuid')
+        url = '/v1/containers/%s/%s/' % (container_uuid, 'commit')
+        cmd = {'repository': 'repo', 'tag': 'tag'}
+        response = self.app.post(url, cmd)
+        self.assertEqual(202, response.status_int)
+        mock_container_commit.assert_called_once_with(
+            mock.ANY, test_container_obj, cmd['repository'], cmd['tag'])
+
+    def test_commit_by_uuid_invalid_state(self):
+        uuid = uuidutils.generate_uuid()
+        cmd = {'repository': 'repo', 'tag': 'tag'}
+        utils.create_test_container(context=self.context,
+                                    uuid=uuid, status='Error')
+        with self.assertRaisesRegexp(
+                AppError, "Cannot commit container %s in Error state" % uuid):
+            self.app.post('/v1/containers/%s/commit/' % uuid, cmd)
+
 
 class TestContainerEnforcement(api_base.FunctionalTest):
 

zun/tests/unit/compute/test_compute_manager.py

@@ -14,7 +14,7 @@
 
 import mock
 
-
+from io import StringIO
 from zun.common import consts
 from zun.common import exception
 from zun.compute import manager
@@ -525,3 +525,26 @@ class TestManager(base.TestCase):
         self.assertRaises(exception.DockerError,
                           self.compute_manager.container_exec_resize,
                           self.context, 'fake_exec_id', "100", "100")
+
+    @mock.patch('zun.image.driver.upload_image')
+    @mock.patch.object(fake_driver, 'get_image')
+    @mock.patch.object(fake_driver, 'commit')
+    def test_container_commit(self, mock_commit,
+                              mock_get_image, mock_upload_image):
+        container = Container(self.context, **utils.get_test_container())
+        mock_get_image_response = mock.MagicMock()
+        mock_get_image_response.data = StringIO().read()
+        mock_get_image.return_value = mock_get_image_response
+        mock_upload_image.return_value = mock.MagicMock()
+
+        self.compute_manager._do_container_commit(self.context,
+                                                  container, 'repo', 'tag')
+        mock_commit.assert_called_once_with(container, 'repo', 'tag')
+
+    @mock.patch.object(fake_driver, 'commit')
+    def test_container_commit_failed(self, mock_commit):
+        container = Container(self.context, **utils.get_test_container())
+        mock_commit.side_effect = exception.DockerError
+        self.assertRaises(exception.DockerError,
+                          self.compute_manager._do_container_commit,
+                          self.context, container, 'repo', 'tag')

zun/tests/unit/container/docker/test_docker_driver.py

@@ -61,6 +61,11 @@ class TestDockerDriver(base.DriverTestCase):
         self.driver.inspect_image(mock_image)
         self.mock_docker.inspect_image.assert_called_once_with(mock_image)
 
+    def test_get_image(self):
+        self.mock_docker.get_image = mock.Mock()
+        self.driver.get_image(name='image_name')
+        self.mock_docker.get_image.assert_called_once_with('image_name')
+
     def test_load_image(self):
         self.mock_docker.load_image = mock.Mock()
         mock_open_file = mock.mock_open()
@@ -336,6 +341,13 @@ class TestDockerDriver(base.DriverTestCase):
         self.mock_docker.resize.assert_called_once_with(
             mock_container.container_id, 100, 100)
 
+    def test_commit(self):
+        self.mock_docker.commit = mock.Mock()
+        mock_container = mock.MagicMock()
+        self.driver.commit(mock_container, "repo", "tag")
+        self.mock_docker.commit.assert_called_once_with(
+            mock_container.container_id, "repo", "tag")
+
     @mock.patch('zun.network.kuryr_network.KuryrNetwork'
                 '.connect_container_to_network')
     @mock.patch('zun.container.docker.driver.DockerDriver.get_sandbox_name')

zun/tests/unit/container/fake_driver.py

@@ -28,6 +28,9 @@ class FakeDriver(driver.ContainerDriver):
     def inspect_image(self, image):
         pass
 
+    def get_image(self, name):
+        pass
+
     def images(self, repo, **kwargs):
         pass
 
@@ -105,3 +108,7 @@ class FakeDriver(driver.ContainerDriver):
     @check_container_id
     def update(self, container):
         pass
+
+    @check_container_id
+    def commit(self, container, repository, tag):
+        pass

zun/tests/unit/image/glance/test_driver.py

@@ -135,3 +135,26 @@ class TestDriver(base.BaseTestCase):
         self.assertRaises(exception.ZunException, self.driver.search_image,
                           None, 'image', None, False)
         self.assertTrue(mock_find_images.called)
+
+    @mock.patch('zun.image.glance.utils.create_image')
+    def test_create_image(self, mock_create_image):
+        image_meta = mock.MagicMock()
+        image_meta.id = '1234'
+        mock_create_image.return_value = [image_meta]
+        ret = self.driver.create_image(None, 'image')
+        self.assertEqual(1, len(ret))
+        self.assertTrue(mock_create_image.called)
+
+    @mock.patch.object(driver.GlanceDriver, 'update_image')
+    @mock.patch('zun.image.glance.utils.update_image_tags')
+    @mock.patch('zun.image.glance.utils.update_image_format')
+    def test_update_image(self, mock_update_image_format,
+                          mock_update_image_tags, mock_update_image):
+        image_meta = mock.MagicMock()
+        image_meta.id = '1234'
+        mock_update_image_tags.return_value = [image_meta]
+        mock_update_image_format.return_value = [image_meta]
+        mock_update_image.return_value = [image_meta]
+        ret = self.driver.update_image(None, 'id', container_format='docker')
+        self.assertEqual(1, len(ret))
+        self.assertTrue(mock_update_image.called)