Upload image via interop import if needed

We have support for interop import already, but it's not stitched
in to the image upload call. Add support for using it.

We're adding this as a fallback for the case that PUT is turned off
due to operator concerns about the need for scratch space. That
means we don't want existing users to all of a sudden start uploading
via import. However, if they do want to use import, we want that
to work, so there is now a flag.

Change-Id: I26e7ba5704d58a21f7ae2011e8c21e9b9310751a

openstack/image/_base_proxy.py

@@ -41,6 +41,7 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
             allow_duplicates=False, meta=None,
             wait=False, timeout=3600,
             data=None, validate_checksum=False,
+            use_import=False,
             **kwargs):
         """Upload an image.
 
@@ -78,6 +79,11 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
             compares return value with the one calculated or passed into this
             call. If value does not match - raises exception. Default is
             'false'
+        :param bool use_import: Use the interoperable image import mechanism
+            to import the image. This defaults to false because it is harder on
+            the target cloud so should only be used when needed, such as when
+            the user needs the cloud to transform image format. If the cloud
+            has disabled direct uploads, this will default to true.
 
         Additional kwargs will be passed to the image creation as additional
         metadata for the image and will have all values converted to string
@@ -170,6 +176,7 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
                 name, filename=filename, data=data, meta=meta,
                 wait=wait, timeout=timeout,
                 validate_checksum=validate_checksum,
+                use_import=use_import,
                 **image_kwargs)
         else:
             image_kwargs['name'] = name
@@ -183,7 +190,7 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
 
     @abc.abstractmethod
     def _upload_image(self, name, filename, data, meta, wait, timeout,
-                      validate_checksum=True,
+                      validate_checksum=True, use_import=False,
                       **image_kwargs):
         pass
 

openstack/image/v1/_proxy.py

@@ -12,6 +12,7 @@
 import warnings
 
 from openstack.cloud import exc
+from openstack import exceptions
 from openstack.image import _base_proxy
 from openstack.image.v1 import image as _image
 
@@ -42,7 +43,13 @@ class Proxy(_base_proxy.BaseImageProxy):
         return self._create(_image.Image, **attrs)
 
     def _upload_image(
-            self, name, filename, data, meta, wait, timeout, **image_kwargs):
+        self, name, filename, data, meta, wait, timeout,
+        use_import=False,
+        **image_kwargs,
+    ):
+        if use_import:
+            raise exceptions.SDKException(
+                "Glance v1 does not support image import")
         # NOTE(mordred) wait and timeout parameters are unused, but
         # are present for ease at calling site.
         if filename and not data:

openstack/image/v2/_proxy.py

@@ -150,6 +150,7 @@ class Proxy(_base_proxy.BaseImageProxy):
 
     def _upload_image(self, name, filename=None, data=None, meta=None,
                       wait=False, timeout=None, validate_checksum=True,
+                      use_import=False,
                       **kwargs):
         # We can never have nice things. Glance v1 took "is_public" as a
         # boolean. Glance v2 takes "visibility". If the user gives us
@@ -165,6 +166,12 @@ class Proxy(_base_proxy.BaseImageProxy):
         try:
             # This makes me want to die inside
             if self._connection.image_api_use_tasks:
+                if use_import:
+                    raise exceptions.SDKException(
+                        "The Glance Task API and Import API are"
+                        " mutually exclusive. Either disable"
+                        " image_api_use_tasks in config, or"
+                        " do not request using import")
                 return self._upload_image_task(
                     name, filename, data=data, meta=meta,
                     wait=wait, timeout=timeout, **kwargs)
@@ -172,6 +179,7 @@ class Proxy(_base_proxy.BaseImageProxy):
                 return self._upload_image_put(
                     name, filename, data=data, meta=meta,
                     validate_checksum=validate_checksum,
+                    use_import=use_import,
                     **kwargs)
         except exceptions.SDKException:
             self.log.debug("Image creation failed", exc_info=True)
@@ -196,8 +204,10 @@ class Proxy(_base_proxy.BaseImageProxy):
         return ret
 
     def _upload_image_put(
-            self, name, filename, data, meta,
-            validate_checksum, **image_kwargs):
+        self, name, filename, data, meta,
+        validate_checksum, use_import=False,
+        **image_kwargs,
+    ):
         if filename and not data:
             image_data = open(filename, 'rb')
         else:
@@ -211,10 +221,28 @@ class Proxy(_base_proxy.BaseImageProxy):
         image = self._create(_image.Image, **image_kwargs)
 
         image.data = image_data
+        supports_import = (
+            image.image_import_methods
+            and 'glance-direct' in image.image_import_methods
+        )
+        if use_import and not supports_import:
+            raise exceptions.SDKException(
+                "Importing image was requested but the cloud does not"
+                " support the image import method.")
 
         try:
-            response = image.upload(self)
-            exceptions.raise_from_response(response)
+            if not use_import:
+                try:
+                    response = image.upload(self)
+                    exceptions.raise_from_response(response)
+                except Exception:
+                    if not supports_import:
+                        raise
+                    use_import = True
+            if use_import:
+                image.stage(self)
+                image.import_image(self)
+
             # image_kwargs are flat here
             md5 = image_kwargs.get(self._IMAGE_MD5_KEY)
             sha256 = image_kwargs.get(self._IMAGE_SHA256_KEY)

openstack/image/v2/image.py

@@ -276,8 +276,6 @@ class Image(resource.Resource, resource.TagMixin, _download.DownloadMixin):
         if uri:
             if method == 'web-download':
                 json['method']['uri'] = uri
-            elif method == 'glance-direct':
-                json['method']['uri'] = uri
             else:
                 raise exceptions.InvalidRequest('URI is only supported with '
                                                 'method: "web-download"')
@@ -286,6 +284,14 @@ class Image(resource.Resource, resource.TagMixin, _download.DownloadMixin):
             headers = {'X-Image-Meta-Store': store.id}
         session.post(url, json=json, headers=headers)
 
+    def _consume_header_attrs(self, attrs):
+        self.image_import_methods = []
+        _image_import_methods = attrs.pop('OpenStack-image-import-methods', '')
+        if _image_import_methods:
+            self.image_import_methods = _image_import_methods.split(',')
+
+        return super()._consume_header_attrs(attrs)
+
     def _prepare_request(self, requires_id=None, prepend_key=False,
                          patch=False, base_path=None, **kwargs):
         request = super(Image, self)._prepare_request(requires_id=requires_id,

openstack/tests/unit/cloud/test_image.py

@@ -22,6 +22,8 @@ from openstack.cloud import meta
 from openstack.tests import fakes
 from openstack.tests.unit import base
 
+IMPORT_METHODS = 'glance-direct,web-download'
+
 
 class BaseTestImage(base.TestCase):
 
@@ -314,7 +316,7 @@ class TestImage(BaseTestImage):
             self.cloud.list_images())
         self.assert_calls()
 
-    def test_create_image_put_v2(self):
+    def test_create_image_put_v2_no_import(self):
         self.cloud.image_api_use_tasks = False
 
         self.register_uris([
@@ -382,6 +384,238 @@ class TestImage(BaseTestImage):
         self.assertEqual(self.adapter.request_history[7].text.read(),
                          self.output)
 
+    def test_create_image_put_v2_import_supported(self):
+        self.cloud.image_api_use_tasks = False
+
+        self.register_uris([
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_name],
+                     base_url_append='v2'),
+                 status_code=404),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'],
+                     base_url_append='v2',
+                     qs_elements=['name=' + self.image_name]),
+                 validate=dict(),
+                 json={'images': []}),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'],
+                     base_url_append='v2',
+                     qs_elements=['os_hidden=True']),
+                 json={'images': []}),
+            dict(method='POST',
+                 uri=self.get_mock_url(
+                     'image', append=['images'], base_url_append='v2'),
+                 json=self.fake_image_dict,
+                 headers={
+                     'OpenStack-image-import-methods': IMPORT_METHODS,
+                 },
+                 validate=dict(
+                     json={
+                         u'container_format': u'bare',
+                         u'disk_format': u'qcow2',
+                         u'name': self.image_name,
+                         u'owner_specified.openstack.md5':
+                             self.fake_image_dict[
+                                 'owner_specified.openstack.md5'],
+                         u'owner_specified.openstack.object': self.object_name,
+                         u'owner_specified.openstack.sha256':
+                             self.fake_image_dict[
+                                 'owner_specified.openstack.sha256'],
+                         u'visibility': u'private',
+                         u'tags': [u'tag1', u'tag2']})
+                 ),
+            dict(method='PUT',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_id, 'file'],
+                     base_url_append='v2'),
+                 request_headers={'Content-Type': 'application/octet-stream'}),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.fake_image_dict['id']],
+                     base_url_append='v2'
+                 ),
+                 json=self.fake_image_dict),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'], base_url_append='v2'),
+                 complete_qs=True,
+                 json=self.fake_search_return)
+        ])
+
+        self.cloud.create_image(
+            self.image_name, self.imagefile.name, wait=True, timeout=1,
+            tags=['tag1', 'tag2'],
+            is_public=False, validate_checksum=True)
+
+        self.assert_calls()
+        self.assertEqual(self.adapter.request_history[7].text.read(),
+                         self.output)
+
+    def test_create_image_use_import(self):
+        self.cloud.image_api_use_tasks = False
+
+        self.register_uris([
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_name],
+                     base_url_append='v2'),
+                 status_code=404),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'],
+                     base_url_append='v2',
+                     qs_elements=['name=' + self.image_name]),
+                 validate=dict(),
+                 json={'images': []}),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'],
+                     base_url_append='v2',
+                     qs_elements=['os_hidden=True']),
+                 json={'images': []}),
+            dict(method='POST',
+                 uri=self.get_mock_url(
+                     'image', append=['images'], base_url_append='v2'),
+                 json=self.fake_image_dict,
+                 headers={
+                     'OpenStack-image-import-methods': IMPORT_METHODS,
+                 },
+                 validate=dict(
+                     json={
+                         u'container_format': u'bare',
+                         u'disk_format': u'qcow2',
+                         u'name': self.image_name,
+                         u'owner_specified.openstack.md5':
+                             self.fake_image_dict[
+                                 'owner_specified.openstack.md5'],
+                         u'owner_specified.openstack.object': self.object_name,
+                         u'owner_specified.openstack.sha256':
+                             self.fake_image_dict[
+                                 'owner_specified.openstack.sha256'],
+                         u'visibility': u'private',
+                         u'tags': [u'tag1', u'tag2']})
+                 ),
+            dict(method='PUT',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_id, 'stage'],
+                     base_url_append='v2'),
+                 request_headers={'Content-Type': 'application/octet-stream'}),
+            dict(method='POST',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_id, 'import'],
+                     base_url_append='v2'),
+                 json={'method': {'name': 'glance-direct'}}),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.fake_image_dict['id']],
+                     base_url_append='v2'
+                 ),
+                 json=self.fake_image_dict),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'], base_url_append='v2'),
+                 complete_qs=True,
+                 json=self.fake_search_return)
+        ])
+
+        self.cloud.create_image(
+            self.image_name, self.imagefile.name, wait=True, timeout=1,
+            tags=['tag1', 'tag2'],
+            is_public=False, validate_checksum=True,
+            use_import=True,
+        )
+
+        self.assert_calls()
+        self.assertEqual(self.adapter.request_history[7].text.read(),
+                         self.output)
+
+    def test_create_image_import_fallback(self):
+        self.cloud.image_api_use_tasks = False
+
+        self.register_uris([
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_name],
+                     base_url_append='v2'),
+                 status_code=404),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'],
+                     base_url_append='v2',
+                     qs_elements=['name=' + self.image_name]),
+                 validate=dict(),
+                 json={'images': []}),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'],
+                     base_url_append='v2',
+                     qs_elements=['os_hidden=True']),
+                 json={'images': []}),
+            dict(method='POST',
+                 uri=self.get_mock_url(
+                     'image', append=['images'], base_url_append='v2'),
+                 json=self.fake_image_dict,
+                 headers={
+                     'OpenStack-image-import-methods': IMPORT_METHODS,
+                 },
+                 validate=dict(
+                     json={
+                         u'container_format': u'bare',
+                         u'disk_format': u'qcow2',
+                         u'name': self.image_name,
+                         u'owner_specified.openstack.md5':
+                             self.fake_image_dict[
+                                 'owner_specified.openstack.md5'],
+                         u'owner_specified.openstack.object': self.object_name,
+                         u'owner_specified.openstack.sha256':
+                             self.fake_image_dict[
+                                 'owner_specified.openstack.sha256'],
+                         u'visibility': u'private',
+                         u'tags': [u'tag1', u'tag2']})
+                 ),
+            dict(method='PUT',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_id, 'file'],
+                     base_url_append='v2'),
+                 request_headers={'Content-Type': 'application/octet-stream'},
+                 status_code=403),
+            dict(method='PUT',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_id, 'stage'],
+                     base_url_append='v2'),
+                 request_headers={'Content-Type': 'application/octet-stream'}),
+            dict(method='POST',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.image_id, 'import'],
+                     base_url_append='v2'),
+                 json={'method': {'name': 'glance-direct'}}),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images', self.fake_image_dict['id']],
+                     base_url_append='v2'
+                 ),
+                 json=self.fake_image_dict),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'image', append=['images'], base_url_append='v2'),
+                 complete_qs=True,
+                 json=self.fake_search_return)
+        ])
+
+        self.cloud.create_image(
+            self.image_name, self.imagefile.name, wait=True, timeout=1,
+            tags=['tag1', 'tag2'],
+            is_public=False, validate_checksum=True,
+        )
+
+        self.assert_calls()
+        self.assertEqual(self.adapter.request_history[7].text.read(),
+                         self.output)
+
     def test_create_image_task(self):
         self.cloud.image_api_use_tasks = True
         endpoint = self.cloud._object_store_client.get_endpoint()

openstack/tests/unit/image/v2/test_proxy.py

@@ -132,7 +132,9 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
                                               'd8262cd4f54963f0c93082d8dcf33'
                                               '4d4c78',
                 self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
-            timeout=3600, validate_checksum=True, wait=False)
+            timeout=3600, validate_checksum=True,
+            use_import=False,
+            wait=False)
 
     def test_image_create_validate_checksum_data_not_binary(self):
         self.assertRaises(
@@ -161,7 +163,9 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
                 self.proxy._IMAGE_MD5_KEY: '',
                 self.proxy._IMAGE_SHA256_KEY: '',
                 self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
-            timeout=3600, validate_checksum=False, wait=False)
+            timeout=3600, validate_checksum=False,
+            use_import=False,
+            wait=False)
 
     def test_image_create_without_filename(self):
         self.proxy._create_image = mock.Mock()

releasenotes/notes/image-import-support-97052cdbc8ce449b.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Added support for using the image import feature when creating an
+    image. SDK will now fall back to using image import if there is an
+    error during PUT.