Support glance web-download

This adds a new resource to support import of glance web-download. It
replaces the old image source using glance v1.

Story: #2004772
Task: #28891
Change-Id: Iae66aa82d6b90738e4f32ee254b9f0c8275a8c87

heat/engine/resources/openstack/glance/image.py

@@ -19,6 +19,227 @@ from heat.engine import resource
 from heat.engine import support
 
 
+class GlanceWebImage(resource.Resource):
+    """A resource managing images in Glance using web-download import.
+
+    This provides image support for recent Glance installation.
+    """
+
+    support_status = support.SupportStatus(version='12.0.0')
+
+    PROPERTIES = (
+        NAME, IMAGE_ID, MIN_DISK, MIN_RAM, PROTECTED,
+        DISK_FORMAT, CONTAINER_FORMAT, LOCATION, TAGS,
+        ARCHITECTURE, KERNEL_ID, OS_DISTRO, OS_VERSION, OWNER,
+        VISIBILITY, RAMDISK_ID
+    ) = (
+        'name', 'id', 'min_disk', 'min_ram', 'protected',
+        'disk_format', 'container_format', 'location', 'tags',
+        'architecture', 'kernel_id', 'os_distro', 'os_version', 'owner',
+        'visibility', 'ramdisk_id'
+    )
+
+    glance_id_pattern = ('^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}'
+                         '-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$')
+
+    properties_schema = {
+        NAME: properties.Schema(
+            properties.Schema.STRING,
+            _('Name for the image. The name of an image is not '
+              'unique to a Image Service node.')
+        ),
+        IMAGE_ID: properties.Schema(
+            properties.Schema.STRING,
+            _('The image ID. Glance will generate a UUID if not specified.')
+        ),
+        MIN_DISK: properties.Schema(
+            properties.Schema.INTEGER,
+            _('Amount of disk space (in GB) required to boot image. '
+              'Default value is 0 if not specified '
+              'and means no limit on the disk size.'),
+            constraints=[
+                constraints.Range(min=0),
+            ],
+            default=0
+        ),
+        MIN_RAM: properties.Schema(
+            properties.Schema.INTEGER,
+            _('Amount of ram (in MB) required to boot image. Default value '
+              'is 0 if not specified and means no limit on the ram size.'),
+            constraints=[
+                constraints.Range(min=0),
+            ],
+            default=0
+        ),
+        PROTECTED: properties.Schema(
+            properties.Schema.BOOLEAN,
+            _('Whether the image can be deleted. If the value is True, '
+              'the image is protected and cannot be deleted.'),
+            default=False
+        ),
+        DISK_FORMAT: properties.Schema(
+            properties.Schema.STRING,
+            _('Disk format of image.'),
+            required=True,
+            constraints=[
+                constraints.AllowedValues(
+                    ['ami', 'ari', 'aki', 'vhd', 'vhdx', 'vmdk', 'raw',
+                     'qcow2', 'vdi', 'iso', 'ploop'])
+            ]
+        ),
+        CONTAINER_FORMAT: properties.Schema(
+            properties.Schema.STRING,
+            _('Container format of image.'),
+            required=True,
+            constraints=[
+                constraints.AllowedValues([
+                    'ami', 'ari', 'aki', 'bare', 'ovf', 'ova', 'docker'])
+            ]
+        ),
+        LOCATION: properties.Schema(
+            properties.Schema.STRING,
+            _('URL where the data for this image already resides. For '
+              'example, if the image data is stored in swift, you could '
+              'specify "swift://example.com/container/obj".'),
+            required=True,
+        ),
+        TAGS: properties.Schema(
+            properties.Schema.LIST,
+            _('List of image tags.'),
+            update_allowed=True,
+        ),
+        ARCHITECTURE: properties.Schema(
+            properties.Schema.STRING,
+            _('Operating system architecture.'),
+            update_allowed=True,
+        ),
+        KERNEL_ID: properties.Schema(
+            properties.Schema.STRING,
+            _('ID of image stored in Glance that should be used as '
+              'the kernel when booting an AMI-style image.'),
+            update_allowed=True,
+            constraints=[
+                constraints.AllowedPattern(glance_id_pattern)
+            ]
+        ),
+        OS_DISTRO: properties.Schema(
+            properties.Schema.STRING,
+            _('The common name of the operating system distribution '
+              'in lowercase.'),
+            update_allowed=True,
+        ),
+        OS_VERSION: properties.Schema(
+            properties.Schema.STRING,
+            _('Operating system version as specified by the distributor.'),
+            update_allowed=True,
+        ),
+        OWNER: properties.Schema(
+            properties.Schema.STRING,
+            _('Owner of the image.'),
+            update_allowed=True,
+        ),
+        VISIBILITY: properties.Schema(
+            properties.Schema.STRING,
+            _('Scope of image accessibility.'),
+            update_allowed=True,
+            default='private',
+            constraints=[
+                constraints.AllowedValues(
+                    ['public', 'private', 'community', 'shared'])
+            ]
+        ),
+        RAMDISK_ID: properties.Schema(
+            properties.Schema.STRING,
+            _('ID of image stored in Glance that should be used as '
+              'the ramdisk when booting an AMI-style image.'),
+            update_allowed=True,
+            constraints=[
+                constraints.AllowedPattern(glance_id_pattern)
+            ]
+        )
+    }
+
+    default_client_name = 'glance'
+
+    entity = 'images'
+
+    def handle_create(self):
+        args = dict((k, v) for k, v in self.properties.items()
+                    if v is not None)
+
+        location = args.pop(self.LOCATION)
+        images = self.client().images
+        image_id = images.create(
+            **args).id
+        self.resource_id_set(image_id)
+
+        images.image_import(image_id, method='web-download', uri=location)
+
+        return image_id
+
+    def check_create_complete(self, image_id):
+        image = self.client().images.get(image_id)
+        return image.status == 'active'
+
+    def handle_update(self, json_snippet, tmpl_diff, prop_diff):
+        if prop_diff and self.TAGS in prop_diff:
+            existing_tags = self.properties.get(self.TAGS) or []
+            diff_tags = prop_diff.pop(self.TAGS) or []
+
+            new_tags = set(diff_tags) - set(existing_tags)
+            for tag in new_tags:
+                self.client().image_tags.update(
+                    self.resource_id,
+                    tag)
+
+            removed_tags = set(existing_tags) - set(diff_tags)
+            for tag in removed_tags:
+                with self.client_plugin().ignore_not_found:
+                    self.client().image_tags.delete(
+                        self.resource_id,
+                        tag)
+
+        images = self.client().images
+
+        images.update(self.resource_id, **prop_diff)
+
+    def validate(self):
+        super(GlanceWebImage, self).validate()
+        container_format = self.properties[self.CONTAINER_FORMAT]
+        if (container_format in ['ami', 'ari', 'aki']
+                and self.properties[self.DISK_FORMAT] != container_format):
+            msg = _("Invalid mix of disk and container formats. When "
+                    "setting a disk or container format to one of 'aki', "
+                    "'ari', or 'ami', the container and disk formats must "
+                    "match.")
+            raise exception.StackValidationFailed(message=msg)
+
+    def get_live_resource_data(self):
+        image_data = super(GlanceWebImage, self).get_live_resource_data()
+        if image_data.get('status') in ('deleted', 'killed'):
+                raise exception.EntityNotFound(entity='Resource',
+                                               name=self.name)
+        return image_data
+
+    def parse_live_resource_data(self, resource_properties, resource_data):
+        image_reality = {}
+
+        for key in self.PROPERTIES:
+            if key == self.LOCATION:
+                continue
+            if key == self.IMAGE_ID:
+                if (resource_properties.get(self.IMAGE_ID) is not None or
+                        resource_data.get(self.IMAGE_ID) != self.resource_id):
+                    image_reality.update({self.IMAGE_ID: resource_data.get(
+                        self.IMAGE_ID)})
+                else:
+                    image_reality.update({self.IMAGE_ID: None})
+            else:
+                image_reality.update({key: resource_data.get(key)})
+
+        return image_reality
+
+
 class GlanceImage(resource.Resource):
     """A resource managing images in Glance.
 
@@ -291,5 +512,6 @@ class GlanceImage(resource.Resource):
 
 def resource_mapping():
     return {
-        'OS::Glance::Image': GlanceImage
+        'OS::Glance::Image': GlanceImage,
+        'OS::Glance::WebImage': GlanceWebImage
     }

heat/tests/openstack/glance/test_image.py

@@ -38,7 +38,6 @@ resources:
       min_ram: 512
       protected: False
       location: https://launchpad.net/cirros/cirros-0.3.0-x86_64-disk.img
-      location: https://launchpad.net/cirros/cirros-0.3.0-x86_64-disk.img
       architecture: test_architecture
       kernel_id: 12345678-1234-1234-1234-123456789012
       os_distro: test_distro
@@ -59,6 +58,41 @@ resources:
       location: https://launchpad.net/cirros/cirros-0.3.0-x86_64-disk.img
 '''
 
+image_download_template = '''
+heat_template_version: rocky
+description: This template to define a glance image.
+resources:
+  my_image:
+    type: OS::Glance::WebImage
+    properties:
+      name: cirros_image
+      id: 41f0e60c-ebb4-4375-a2b4-845ae8b9c995
+      disk_format: qcow2
+      container_format: bare
+      min_disk: 10
+      min_ram: 512
+      protected: False
+      location: https://launchpad.net/cirros/cirros-0.3.0-x86_64-disk.img
+      architecture: test_architecture
+      kernel_id: 12345678-1234-1234-1234-123456789012
+      os_distro: test_distro
+      owner: test_owner
+      ramdisk_id: 12345678-1234-1234-1234-123456789012
+'''
+
+image_download_template_validate = '''
+heat_template_version: rocky
+description: This template to define a glance image.
+resources:
+  image:
+    type: OS::Glance::WebImage
+    properties:
+      name: image_validate
+      disk_format: qcow2
+      container_format: bare
+      location: https://launchpad.net/cirros/cirros-0.3.0-x86_64-disk.img
+'''
+
 
 class GlanceImageTest(common.HeatTestCase):
     def setUp(self):
@@ -428,3 +462,345 @@ class GlanceImageTest(common.HeatTestCase):
         self.assertRaises(exception.EntityNotFound,
                           self.my_image.get_live_state,
                           self.my_image.properties)
+
+
+class GlanceWebImageTest(common.HeatTestCase):
+    def setUp(self):
+        super(GlanceWebImageTest, self).setUp()
+
+        self.ctx = utils.dummy_context()
+        tpl = template_format.parse(image_download_template)
+        self.stack = parser.Stack(
+            self.ctx, 'glance_image_test_stack',
+            template.Template(tpl)
+        )
+
+        self.my_image = self.stack['my_image']
+        glance = mock.MagicMock()
+        self.glanceclient = mock.MagicMock()
+        self.my_image.client = glance
+        glance.return_value = self.glanceclient
+        self.images = self.glanceclient.images
+        self.image_tags = self.glanceclient.image_tags
+
+    def _test_validate(self, resource, error_msg):
+        exc = self.assertRaises(exception.StackValidationFailed,
+                                resource.validate)
+        self.assertIn(error_msg, six.text_type(exc))
+
+    def test_invalid_min_disk(self):
+        # invalid 'min_disk'
+        tpl = template_format.parse(image_template_validate)
+        stack = parser.Stack(
+            self.ctx, 'glance_image_stack_validate',
+            template.Template(tpl)
+        )
+        image = stack['image']
+        props = stack.t.t['resources']['image']['properties'].copy()
+        props['min_disk'] = -1
+        image.t = image.t.freeze(properties=props)
+        image.reparse()
+        error_msg = ('Property error: resources.image.properties.min_disk: '
+                     '-1 is out of range (min: 0, max: None)')
+        self._test_validate(image, error_msg)
+
+    def test_invalid_min_ram(self):
+        # invalid 'min_ram'
+        tpl = template_format.parse(image_template_validate)
+        stack = parser.Stack(
+            self.ctx, 'glance_image_stack_validate',
+            template.Template(tpl)
+        )
+        image = stack['image']
+        props = stack.t.t['resources']['image']['properties'].copy()
+        props['min_ram'] = -1
+        image.t = image.t.freeze(properties=props)
+        image.reparse()
+        error_msg = ('Property error: resources.image.properties.min_ram: '
+                     '-1 is out of range (min: 0, max: None)')
+        self._test_validate(image, error_msg)
+
+    def test_miss_disk_format(self):
+        # miss disk_format
+        tpl = template_format.parse(image_template_validate)
+        stack = parser.Stack(
+            self.ctx, 'glance_image_stack_validate',
+            template.Template(tpl)
+        )
+        image = stack['image']
+        props = stack.t.t['resources']['image']['properties'].copy()
+        del props['disk_format']
+        image.t = image.t.freeze(properties=props)
+        image.reparse()
+        error_msg = 'Property disk_format not assigned'
+        self._test_validate(image, error_msg)
+
+    def test_invalid_disk_format(self):
+        # invalid disk_format
+        tpl = template_format.parse(image_template_validate)
+        stack = parser.Stack(
+            self.ctx, 'glance_image_stack_validate',
+            template.Template(tpl)
+        )
+        image = stack['image']
+        props = stack.t.t['resources']['image']['properties'].copy()
+        props['disk_format'] = 'incorrect_format'
+        image.t = image.t.freeze(properties=props)
+        image.reparse()
+        error_msg = ('Property error: '
+                     'resources.image.properties.disk_format: '
+                     '"incorrect_format" is not an allowed value '
+                     '[ami, ari, aki, vhd, vmdk, raw, qcow2, vdi, iso]')
+        self._test_validate(image, error_msg)
+
+    def test_miss_container_format(self):
+        # miss container_format
+        tpl = template_format.parse(image_template_validate)
+        stack = parser.Stack(
+            self.ctx, 'glance_image_stack_validate',
+            template.Template(tpl)
+        )
+        image = stack['image']
+        props = stack.t.t['resources']['image']['properties'].copy()
+        del props['container_format']
+        image.t = image.t.freeze(properties=props)
+        image.reparse()
+        error_msg = 'Property container_format not assigned'
+        self._test_validate(image, error_msg)
+
+    def test_invalid_container_format(self):
+        # invalid container_format
+        tpl = template_format.parse(image_template_validate)
+        stack = parser.Stack(
+            self.ctx, 'glance_image_stack_validate',
+            template.Template(tpl)
+        )
+        image = stack['image']
+        props = stack.t.t['resources']['image']['properties'].copy()
+        props['container_format'] = 'incorrect_format'
+        image.t = image.t.freeze(properties=props)
+        image.reparse()
+        error_msg = ('Property error: '
+                     'resources.image.properties.container_format: '
+                     '"incorrect_format" is not an allowed value '
+                     '[ami, ari, aki, bare, ova, ovf]')
+        self._test_validate(image, error_msg)
+
+    def test_miss_location(self):
+        # miss location
+        tpl = template_format.parse(image_template_validate)
+        stack = parser.Stack(
+            self.ctx, 'glance_image_stack_validate',
+            template.Template(tpl)
+        )
+        image = stack['image']
+        props = stack.t.t['resources']['image']['properties'].copy()
+        del props['location']
+        image.t = image.t.freeze(properties=props)
+        image.reparse()
+        error_msg = 'Property location not assigned'
+        self._test_validate(image, error_msg)
+
+    def test_invalid_disk_container_mix(self):
+        tpl = template_format.parse(image_template_validate)
+        stack = parser.Stack(
+            self.ctx, 'glance_image_stack_validate',
+            template.Template(tpl)
+        )
+        image = stack['image']
+        props = stack.t.t['resources']['image']['properties'].copy()
+        props['disk_format'] = 'raw'
+        props['container_format'] = 'ari'
+        image.t = image.t.freeze(properties=props)
+        image.reparse()
+        error_msg = ("Invalid mix of disk and container formats. When "
+                     "setting a disk or container format to one of 'aki', "
+                     "'ari', or 'ami', the container and disk formats must "
+                     "match.")
+        self._test_validate(image, error_msg)
+
+    def test_image_handle_create(self):
+        value = mock.MagicMock()
+        image_id = '41f0e60c-ebb4-4375-a2b4-845ae8b9c995'
+        value.id = image_id
+        self.images.create.return_value = value
+        self.image_tags.update.return_value = None
+        props = self.stack.t.t['resources']['my_image']['properties'].copy()
+        props['tags'] = ['tag1']
+        self.my_image.t = self.my_image.t.freeze(properties=props)
+        self.my_image.reparse()
+        self.my_image.handle_create()
+
+        self.assertEqual(image_id, self.my_image.resource_id)
+        # assert that no tags pass when image create
+        self.images.create.assert_called_once_with(
+            architecture='test_architecture',
+            container_format=u'bare',
+            disk_format=u'qcow2',
+            id=u'41f0e60c-ebb4-4375-a2b4-845ae8b9c995',
+            kernel_id='12345678-1234-1234-1234-123456789012',
+            os_distro='test_distro',
+            ramdisk_id='12345678-1234-1234-1234-123456789012',
+            visibility='private',
+            min_disk=10,
+            min_ram=512,
+            name=u'cirros_image',
+            protected=False,
+            owner=u'test_owner',
+            tags=['tag1'],
+        )
+
+    def _handle_update_tags(self, prop_diff):
+        self.my_image.handle_update(json_snippet=None,
+                                    tmpl_diff=None,
+                                    prop_diff=prop_diff)
+
+        self.image_tags.update.assert_called_once_with(
+            self.my_image.resource_id,
+            'tag2'
+        )
+        self.image_tags.delete.assert_called_once_with(
+            self.my_image.resource_id,
+            'tag1'
+        )
+
+    def test_image_handle_update(self):
+        self.my_image.resource_id = '477e8273-60a7-4c41-b683-fdb0bc7cd151'
+        prop_diff = {
+            'architecture': 'test_architecture',
+            'kernel_id': '12345678-1234-1234-1234-123456789012',
+            'os_distro': 'test_distro',
+            'owner': 'test_owner',
+            'ramdisk_id': '12345678-1234-1234-1234-123456789012'}
+
+        self.my_image.handle_update(json_snippet=None,
+                                    tmpl_diff=None,
+                                    prop_diff=prop_diff)
+        self.images.update.assert_called_once_with(
+            self.my_image.resource_id,
+            architecture='test_architecture',
+            kernel_id='12345678-1234-1234-1234-123456789012',
+            os_distro='test_distro',
+            owner='test_owner',
+            ramdisk_id='12345678-1234-1234-1234-123456789012'
+        )
+
+    def test_image_handle_update_tags(self):
+        self.my_image.resource_id = '477e8273-60a7-4c41-b683-fdb0bc7cd151'
+
+        props = self.stack.t.t['resources']['my_image']['properties'].copy()
+        props['tags'] = ['tag1']
+        self.my_image.t = self.my_image.t.freeze(properties=props)
+        self.my_image.reparse()
+        prop_diff = {'tags': ['tag2']}
+
+        self._handle_update_tags(prop_diff)
+
+    def test_image_handle_update_remove_tags(self):
+        self.my_image.resource_id = '477e8273-60a7-4c41-b683-fdb0bc7cd151'
+
+        props = self.stack.t.t['resources']['my_image']['properties'].copy()
+        props['tags'] = ['tag1']
+        self.my_image.t = self.my_image.t.freeze(properties=props)
+        self.my_image.reparse()
+        prop_diff = {'tags': None}
+
+        self.my_image.handle_update(json_snippet=None,
+                                    tmpl_diff=None,
+                                    prop_diff=prop_diff)
+
+        self.image_tags.delete.assert_called_once_with(
+            self.my_image.resource_id,
+            'tag1'
+        )
+
+    def test_image_handle_update_tags_delete_not_found(self):
+        self.my_image.resource_id = '477e8273-60a7-4c41-b683-fdb0bc7cd151'
+
+        props = self.stack.t.t['resources']['my_image']['properties'].copy()
+        props['tags'] = ['tag1']
+        self.my_image.t = self.my_image.t.freeze(properties=props)
+        self.my_image.reparse()
+        prop_diff = {'tags': ['tag2']}
+
+        self.image_tags.delete.side_effect = exc.HTTPNotFound()
+
+        self._handle_update_tags(prop_diff)
+
+    def test_image_show_resource_v1(self):
+        self.glanceclient.version = 1.0
+        self.my_image.resource_id = 'test_image_id'
+        image = mock.MagicMock()
+        images = mock.MagicMock()
+        image.to_dict.return_value = {'image': 'info'}
+        images.get.return_value = image
+        self.my_image.client().images = images
+        self.assertEqual({'image': 'info'}, self.my_image.FnGetAtt('show'))
+        images.get.assert_called_once_with('test_image_id')
+
+    def test_image_show_resource_v2(self):
+        self.my_image.resource_id = 'test_image_id'
+        # glance image in v2 is warlock.model object, so it can be
+        # handled via dict(). In test we use easiest analog - dict.
+        image = {"key1": "val1", "key2": "val2"}
+        self.images.get.return_value = image
+        self.glanceclient.version = 2.0
+        self.assertEqual({"key1": "val1", "key2": "val2"},
+                         self.my_image.FnGetAtt('show'))
+        self.images.get.assert_called_once_with('test_image_id')
+
+    def test_image_get_live_state_v2(self):
+        self.glanceclient.version = 2.0
+        self.my_image.resource_id = '1234'
+        images = mock.MagicMock()
+        show_value = {
+            'name': 'test',
+            'disk_format': 'qcow2',
+            'container_format': 'bare',
+            'protected': False,
+            'is_public': False,
+            'min_disk': 0,
+            'min_ram': 0,
+            'id': '41f0e60c-ebb4-4375-a2b4-845ae8b9c995',
+            'tags': [],
+            'architecture': 'test_architecture',
+            'kernel_id': '12345678-1234-1234-1234-123456789012',
+            'os_distro': 'test_distro',
+            'os_version': '1.0',
+            'owner': 'test_owner',
+            'ramdisk_id': '12345678-1234-1234-1234-123456789012',
+            'visibility': 'private'
+        }
+        image = show_value
+        images.get.return_value = image
+        self.my_image.client().images = images
+
+        reality = self.my_image.get_live_state(self.my_image.properties)
+        expected = {
+            'name': 'test',
+            'disk_format': 'qcow2',
+            'container_format': 'bare',
+            'protected': False,
+            'min_disk': 0,
+            'min_ram': 0,
+            'id': '41f0e60c-ebb4-4375-a2b4-845ae8b9c995',
+            'tags': [],
+            'architecture': 'test_architecture',
+            'kernel_id': '12345678-1234-1234-1234-123456789012',
+            'os_distro': 'test_distro',
+            'os_version': '1.0',
+            'owner': 'test_owner',
+            'ramdisk_id': '12345678-1234-1234-1234-123456789012',
+            'visibility': 'private'
+        }
+
+        self.assertEqual(set(expected.keys()), set(reality.keys()))
+        for key in expected:
+            self.assertEqual(expected[key], reality[key])
+
+    def test_get_live_state_resource_is_deleted(self):
+        self.my_image.resource_id = '1234'
+        self.my_image.client().images.get.return_value = {'status': 'deleted'}
+        self.assertRaises(exception.EntityNotFound,
+                          self.my_image.get_live_state,
+                          self.my_image.properties)

releasenotes/notes/glance-web-download-c9d1fd2a6a2cb044.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add a new OS::Glance::WebImage resource supporting the web-download import
+    of Glance v2.