Adding support for container image driver

This patch includes following:
* Glance image driver
* Docker image driver
* Added unit test cases

Whenever a request to create a container with image 'x' comes,
the request will follow as below to pull image from container
image drivers:

1. Look for image locally at location 'images_directory'.
   This is the location where all images tar downloaded from
   glance will be stored. If found, load it.
2. If not found in Step 1, look in 'image_driver_list'. This
   config is the list of image_drivers. Possible options are
   Glance and Docker now.
   Based on the priority defined in 'image_driver_list', pull
   the image from image_driver.
3. Finally create container with loaded image.

Change-Id: Id2230fd5f7f7bc8d517581f69882ac867ba0e59b
Partial-Implements: blueprint glance-integration

zun/common/keystone.py

@@ -32,6 +32,8 @@ CONF.import_group('keystone_authtoken', 'keystonemiddleware.auth_token')
 ka_loading.register_auth_conf_options(CONF, CFG_GROUP)
 ka_loading.register_session_conf_options(CONF, CFG_GROUP)
 CONF.set_default('auth_type', default='password', group=CFG_GROUP)
+CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token',
+                group='keystone_authtoken')
 
 
 class KeystoneClientV3(object):
@@ -44,7 +46,7 @@ class KeystoneClientV3(object):
 
     @property
     def auth_url(self):
-        return CONF.keystone_auth.auth_uri.replace('v2.0', 'v3')
+        return CONF.keystone_authtoken.auth_uri.replace('v2.0', 'v3')
 
     @property
     def auth_token(self):

zun/compute/manager.py

@@ -22,6 +22,7 @@ from zun.common.i18n import _LE
 from zun.common import utils
 from zun.common.utils import translate_exception
 from zun.container import driver
+from zun.image import driver as image_driver
 from zun.objects import fields
 
 
@@ -67,7 +68,7 @@ class Manager(object):
         container.task_state = fields.TaskState.IMAGE_PULLING
         container.save()
         try:
-            self.driver.pull_image(container.image)
+            image_path = image_driver.pull_image(context, container.image)
         except exception.DockerError as e:
             LOG.error(_LE("Error occured while calling docker API: %s"),
                       six.text_type(e))
@@ -81,7 +82,7 @@ class Manager(object):
         container.task_state = fields.TaskState.CONTAINER_CREATING
         container.save()
         try:
-            container = self.driver.create(container)
+            container = self.driver.create(container, image_path)
         except exception.DockerError as e:
             LOG.error(_LE("Error occured while calling docker API: %s"),
                       six.text_type(e))

zun/container/docker/driver.py

@@ -33,12 +33,6 @@ class DockerDriver(driver.ContainerDriver):
     def __init__(self):
         super(DockerDriver, self).__init__()
 
-    def pull_image(self, image):
-        with docker_utils.docker_client() as docker:
-            LOG.debug('Pulling image %s' % image)
-            image_repo, image_tag = docker_utils.parse_docker_image(image)
-            docker.pull(image_repo, tag=image_tag)
-
     def inspect_image(self, image):
         with docker_utils.docker_client() as docker:
             LOG.debug('Inspecting image %s' % image)
@@ -50,9 +44,13 @@ class DockerDriver(driver.ContainerDriver):
             response = docker.images(repo, quiet)
             return response
 
-    def create(self, container):
+    def create(self, container, image_path=None):
         with docker_utils.docker_client() as docker:
             name = container.name
+            if image_path:
+                LOG.debug('Loading local image %s in docker' % container.image)
+                with open(image_path, 'r') as fd:
+                    docker.load_image(fd.read())
             image = container.image
             LOG.debug('Creating container with image %s name %s'
                       % (image, name))

zun/image/docker/driver.py

@@ -0,0 +1,46 @@
+# Copyright 2016 Intel.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from docker import errors
+
+from oslo_log import log as logging
+
+from zun.common import exception
+from zun.common.i18n import _
+from zun.container.docker import utils as docker_utils
+from zun.image import driver
+
+
+LOG = logging.getLogger(__name__)
+
+
+class DockerDriver(driver.ContainerImageDriver):
+    def __init__(self):
+        super(DockerDriver, self).__init__()
+
+    def pull_image(self, context, image_name):
+        with docker_utils.docker_client() as docker:
+            try:
+                LOG.debug('Pulling image from docker %s,'
+                          ' context %s' % (image_name, context))
+                repo, tag = docker_utils.parse_docker_image(image_name)
+                docker.pull(repo, tag=tag)
+            except errors.APIError as api_error:
+                if '404' in str(api_error):
+                    raise exception.ImageNotFound(str(api_error))
+                raise exception.ZunException(str(api_error))
+            except Exception as e:
+                msg = _('Cannot download image from docker: {0}')
+                raise exception.ZunException(msg.format(e))

zun/image/driver.py

@@ -0,0 +1,127 @@
+# Copyright 2016 Intel.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_utils import importutils
+
+from zun.common import exception
+from zun.common.i18n import _
+from zun.common.i18n import _LE
+from zun.common.i18n import _LI
+from zun.image.glance import utils
+
+
+LOG = logging.getLogger(__name__)
+
+image_driver_opts = [
+    cfg.ListOpt('image_driver_list',
+                default=['glance.driver.GlanceDriver'],
+                help="""Defines the list of image driver to use for downloading image.
+
+Possible values:
+
+* ``docker.driver.DockerDriver``
+* ``glance.driver.GlanceDriver``
+
+Services which consume this:
+
+* ``zun-compute``
+
+Interdependencies to other options:
+
+* None
+""")
+]
+CONF = cfg.CONF
+CONF.register_opts(image_driver_opts)
+
+
+def load_image_driver(image_driver=None):
+    """Load a image driver module.
+
+    Load the container image driver module specified by the image_driver
+    configuration option or, if supplied, the driver name supplied as an
+    argument.
+    :param image_driver: container image driver name to override config opt
+    :returns: a ContainerImageDriver instance
+    """
+    if not image_driver:
+        LOG.error(_LE("Container image driver option required, "
+                      "but not specified"))
+        sys.exit(1)
+
+    LOG.info(_LI("Loading container image driver '%s'"), image_driver)
+    try:
+        driver = importutils.import_object(
+            'zun.image.%s' % image_driver)
+        if not isinstance(driver, ContainerImageDriver):
+            raise Exception(_('Expected driver of type: %s') %
+                            str(ContainerImageDriver))
+
+        return driver
+    except ImportError:
+        LOG.exception(_LE("Unable to load the container image driver"))
+        sys.exit(1)
+
+
+def search_image_on_host(context, image_name):
+    LOG.debug('Searching for image %s locally' % image_name)
+    CONF.import_opt('images_directory', 'zun.image.glance.driver',
+                    group='glance')
+    images_directory = CONF.glance.images_directory
+    try:
+        # TODO(mkrai): Change this to search image entry in zun db
+        #              after the image endpoint is merged.
+        image_meta = utils.find_image(context, image_name)
+    except exception.ImageNotFound:
+        return None
+    if image_meta:
+        out_path = os.path.join(images_directory, image_meta.id + '.tar')
+        if os.path.isfile(out_path):
+            return out_path
+        else:
+            return None
+
+
+def pull_image(context, image_name):
+    image_path = search_image_on_host(context, image_name)
+    if image_path:
+        LOG.debug('Found image %s locally.' % image_name)
+        return image_path
+    image_driver_list = CONF.image_driver_list
+    for driver in image_driver_list:
+        try:
+            image_driver = load_image_driver(driver)
+            image = image_driver.pull_image(context, image_name)
+            if image:
+                break
+        except exception.ImageNotFound:
+            pass
+        except Exception as e:
+            LOG.exception(_LE('Unknown exception occured while loading'
+                              ' image : %s'), str(e))
+            raise exception.ZunException(str(e))
+    return image
+
+
+class ContainerImageDriver(object):
+    '''Base class for container image driver.'''
+
+    def pull_image(self, context, image):
+        """Create an image."""
+        raise NotImplementedError()

zun/image/glance/driver.py

@@ -0,0 +1,75 @@
+# Copyright 2016 Intel.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_utils import fileutils
+
+from zun.common import exception
+from zun.common.i18n import _
+from zun.image import driver
+from zun.image.glance import utils
+
+CONF = cfg.CONF
+
+LOG = logging.getLogger(__name__)
+
+glance_opts = [
+    cfg.StrOpt('images_directory',
+               default=None,
+               help='Shared directory where glance images located. If '
+                    'specified, docker will try to load the image from '
+                    'the shared directory by image ID.'),
+    ]
+CONF = cfg.CONF
+opt_group = cfg.OptGroup(name='glance',
+                         title='Glance options for image management')
+CONF.register_group(opt_group)
+CONF.register_opts(glance_opts, opt_group)
+
+
+class GlanceDriver(driver.ContainerImageDriver):
+    def __init__(self):
+        super(GlanceDriver, self).__init__()
+
+    def pull_image(self, context, image_name):
+        LOG.debug('Pulling image from glance %s' % image_name)
+        try:
+            glance = utils.create_glanceclient(context)
+            image_meta = utils.find_image(context, image_name)
+            LOG.debug('Image %s was found in glance, downloading now...'
+                      % image_name)
+            image_chunks = glance.images.data(image_meta.id)
+        except exception.ImageNotFound:
+            LOG.debug('Image %s was not found in glance' % image_name)
+            raise
+        except Exception as e:
+            msg = _('Cannot download image from glance: {0}')
+            raise exception.ZunException(msg.format(e))
+        try:
+            images_directory = CONF.glance.images_directory
+            fileutils.ensure_tree(images_directory)
+            out_path = os.path.join(images_directory, image_meta.id + '.tar')
+            with open(out_path, 'wb') as fd:
+                for chunk in image_chunks:
+                    fd.write(chunk)
+        except Exception as e:
+            msg = _('Error occured while writing image: {0}')
+            raise exception.ZunException(msg.format(e))
+        LOG.debug('Image %s was downloaded to path : %s'
+                  % (image_name, out_path))
+        return out_path

zun/image/glance/utils.py

@@ -0,0 +1,48 @@
+# Copyright 2016 Intel.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_utils import uuidutils
+
+from zun.common import clients
+from zun.common import exception
+from zun.common.i18n import _LE
+
+
+def create_glanceclient(context):
+    """Creates glance client object.
+
+        :param context: context to create client object
+        :returns: Glance client object
+    """
+    osc = clients.OpenStackClients(context)
+    return osc.glance()
+
+
+def find_image(context, image_ident):
+    glance = create_glanceclient(context)
+    if uuidutils.is_uuid_like(image_ident):
+        image_meta = glance.images.get(image_ident)
+    else:
+        filters = {'name': image_ident}
+        matches = list(glance.images.list(filters=filters))
+        if len(matches) == 0:
+            raise exception.ImageNotFound(image=image_ident)
+        if len(matches) > 1:
+            msg = _LE("Multiple images exist with same name "
+                      "%(image_ident)s. Please use the image id "
+                      "instead.") % {'image_ident': image_ident}
+            raise exception.Conflict(msg)
+        image_meta = matches[0]
+    return image_meta

zun/tests/unit/compute/test_compute_manager.py

@@ -59,17 +59,18 @@ class TestManager(base.TestCase):
                           container, 'unpause')
 
     @mock.patch.object(Container, 'save')
-    @mock.patch.object(fake_driver, 'pull_image')
+    @mock.patch('zun.image.driver.pull_image')
     @mock.patch.object(fake_driver, 'create')
     def test_container_create(self, mock_create, mock_pull, mock_save):
         container = Container(self.context, **utils.get_test_container())
+        mock_pull.return_value = 'fake_path'
         self.compute_manager._do_container_create(self.context, container)
         mock_save.assert_called_with()
-        mock_pull.assert_called_once_with(container.image)
-        mock_create.assert_called_once_with(container)
+        mock_pull.assert_called_once_with(self.context, container.image)
+        mock_create.assert_called_once_with(container, 'fake_path')
 
     @mock.patch.object(Container, 'save')
-    @mock.patch.object(fake_driver, 'pull_image')
+    @mock.patch('zun.image.driver.pull_image')
     @mock.patch.object(manager.Manager, '_fail_container')
     def test_container_create_pull_image_failed(self, mock_fail,
                                                 mock_pull, mock_save):

zun/tests/unit/image/docker/test_driver.py

@@ -0,0 +1,70 @@
+# Copyright 2016 Intel.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from docker import errors
+
+from zun.common import exception
+from zun.container.docker import utils
+from zun.image.docker import driver
+from zun.tests import base
+
+
+class TestDriver(base.BaseTestCase):
+    def setUp(self):
+        super(TestDriver, self).setUp()
+        self.driver = driver.DockerDriver()
+        dfc_patcher = mock.patch.object(utils,
+                                        'docker_client')
+        docker_client = dfc_patcher.start()
+        self.dfc_context_manager = docker_client.return_value
+        self.mock_docker = mock.MagicMock()
+        self.dfc_context_manager.__enter__.return_value = self.mock_docker
+        self.addCleanup(dfc_patcher.stop)
+
+    def test_pull_image_success(self):
+        ret = self.driver.pull_image(None, 'test_image')
+        self.assertIsNone(ret)
+        self.mock_docker.pull.assert_called_once_with(
+            'test_image',
+            tag='latest')
+
+    @mock.patch('zun.container.docker.utils.parse_docker_image')
+    def test_pull_image_not_found(self, mock_parse_image):
+        mock_parse_image.return_value = ('repo', 'tag')
+        with mock.patch.object(errors.APIError, '__str__',
+                               return_value='404 Not Found') as mock_init:
+            self.mock_docker.pull = mock.Mock(
+                side_effect=errors.APIError('Error', '', ''))
+            self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
+                              None, 'nonexisting')
+            self.mock_docker.pull.assert_called_once_with(
+                'repo',
+                tag='tag')
+            self.assertEqual(2, mock_init.call_count)
+
+    @mock.patch('zun.container.docker.utils.parse_docker_image')
+    def test_pull_image_exception(self, mock_parse_image):
+        mock_parse_image.return_value = ('repo', 'tag')
+        with mock.patch.object(errors.APIError, '__str__',
+                               return_value='hit error') as mock_init:
+            self.mock_docker.pull = mock.Mock(
+                side_effect=errors.APIError('Error', '', ''))
+            self.assertRaises(exception.ZunException, self.driver.pull_image,
+                              None, 'nonexisting')
+            self.mock_docker.pull.assert_called_once_with(
+                'repo',
+                tag='tag')
+            self.assertEqual(2, mock_init.call_count)

zun/tests/unit/image/glance/test_driver.py

@@ -0,0 +1,63 @@
+# Copyright 2016 Intel.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+import os
+import shutil
+import tempfile
+
+from zun.common import exception
+import zun.conf
+from zun.image.glance import driver
+from zun.tests import base
+
+CONF = zun.conf.CONF
+
+
+class TestDriver(base.BaseTestCase):
+    def setUp(self):
+        super(TestDriver, self).setUp()
+        self.driver = driver.GlanceDriver()
+        self.test_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        # Remove the directory after the test
+        super(TestDriver, self).tearDown()
+        shutil.rmtree(self.test_dir)
+
+    @mock.patch('zun.image.glance.utils.create_glanceclient')
+    def test_pull_image_failure(self, mock_glance):
+        mock_glance.side_effect = Exception
+        self.assertRaises(exception.ZunException, self.driver.pull_image,
+                          None, 'nonexisting')
+
+    @mock.patch('zun.image.glance.utils.create_glanceclient')
+    def test_pull_image_not_found(self, mock_glance):
+        with mock.patch('zun.image.glance.utils.find_image') as mock_find:
+            mock_find.side_effect = exception.ImageNotFound
+            self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
+                              None, 'nonexisting')
+
+    @mock.patch('zun.image.glance.utils.create_glanceclient')
+    @mock.patch('zun.image.glance.utils.find_image')
+    def test_pull_image_found(self, mock_find_image, mock_glance):
+        mock_glance.images.data = mock.MagicMock(return_value='content')
+        image_meta = mock.MagicMock()
+        image_meta.id = '1234'
+        mock_find_image.return_value = image_meta
+        CONF.set_override('images_directory', self.test_dir, group='glance')
+        out_path = os.path.join(self.test_dir, '1234' + '.tar')
+        ret = self.driver.pull_image(None, 'image')
+        self.assertEqual(out_path, ret)
+        self.assertTrue(os.path.isfile(ret))

zun/tests/unit/image/test_driver.py

@@ -0,0 +1,33 @@
+# Copyright 2016 Intel.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import zun.conf
+from zun.image import driver
+from zun.tests import base
+
+CONF = zun.conf.CONF
+
+
+class TestDriver(base.BaseTestCase):
+    def setUp(self):
+        super(TestDriver, self).setUp()
+
+    def test_load_image_driver_failure(self):
+        self.assertRaises(SystemExit, driver.load_image_driver)
+        self.assertRaises(SystemExit, driver.load_image_driver,
+                          'UnknownDriver')
+
+    def test_load_image_driver(self):
+        CONF.set_override('images_directory', None, group='glance')
+        self.assertTrue(driver.load_image_driver, 'glance.GlanceDriver')