Merge "Support configdrive when doing ramdisk deploy with redfish-virtual-media"

doc/source/admin/drivers/redfish.rst

@@ -269,7 +269,11 @@ of ``ACTIVE``.
 
 This initial interface does not support bootloader configuration
 parameter injection, as such the ``[instance_info]/kernel_append_params``
-setting is ignored. Configuration drives are not supported yet.
+setting is ignored.
+
+Configuration drives are supported starting with the Wallaby release
+for nodes that have a free virtual USB slot. The configuration option
+``[deploy]configdrive_use_object_store`` must be set to ``False`` for now.
 
 Layer 3 or DHCP-less ramdisk booting
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/admin/ramdisk-boot.rst

@@ -99,7 +99,8 @@ The intended use case is for advanced scientific and ephemeral workloads
 where the step of writing an image to the local storage is not required
 or desired. As such, this interface does come with several caveats:
 
-* Configuration drives are not supported.
+* Configuration drives are not supported with network boot, only with Redfish
+  virtual media.
 * Disk image contents are not written to the bare metal node.
 * Users and Operators who intend to leverage this interface should
   expect to leverage a metadata service, custom ramdisk images, or the

ironic/drivers/modules/image_utils.py

@@ -13,7 +13,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import base64
 import functools
+import gzip
 import json
 import os
 import shutil
@@ -118,6 +120,23 @@ class ImageHandler(object):
 
             ironic_utils.unlink_without_raise(published_file)
 
+    @classmethod
+    def unpublish_image_for_node(cls, node, prefix='', suffix=''):
+        """Withdraw the image previously made downloadable.
+
+        Depending on ironic settings, removes previously published file
+        from where it has been published - Swift or local HTTP server's
+        document root.
+
+        :param node: the node for which image was published.
+        :param prefix: object name prefix.
+        :param suffix: object name suffix.
+        """
+        name = _get_name(node, prefix=prefix, suffix=suffix)
+        cls(node.driver).unpublish_image(name)
+        LOG.debug('Removed image %(name)s for node %(node)s',
+                  {'node': node.uuid, 'name': name})
+
     def _append_filename_param(self, url, filename):
         """Append 'filename=<file>' parameter to given URL.
 
@@ -205,20 +224,16 @@ class ImageHandler(object):
         return image_url
 
 
-def _get_floppy_image_name(node):
+def _get_name(node, prefix='', suffix=''):
-    """Returns the floppy image name for a given node.
+    """Get an object name for a given node.
 
     :param node: the node for which image name is to be provided.
     """
-    return "image-%s" % node.uuid
+    if prefix:
-
+        name = "%s-%s" % (prefix, node.uuid)
-
+    else:
-def _get_iso_image_name(node):
+        name = node.uuid
-    """Returns the boot iso image name for a given node.
+    return name + suffix
-
-    :param node: the node for which image name is to be provided.
-    """
-    return "boot-%s.iso" % node.uuid
 
 
 def cleanup_iso_image(task):
@@ -226,10 +241,8 @@ def cleanup_iso_image(task):
 
     :param task: A task from TaskManager.
     """
-    iso_object_name = _get_iso_image_name(task.node)
+    ImageHandler.unpublish_image_for_node(task.node, prefix='boot',
-    img_handler = ImageHandler(task.node.driver)
+                                          suffix='.iso')
-
-    img_handler.unpublish_image(iso_object_name)
 
 
 def prepare_floppy_image(task, params=None):
@@ -251,7 +264,7 @@ def prepare_floppy_image(task, params=None):
     :raises: SwiftOperationError, if any operation with Swift fails.
     :returns: image URL for the floppy image.
     """
-    object_name = _get_floppy_image_name(task.node)
+    object_name = _get_name(task.node, prefix='image')
 
     LOG.debug("Trying to create floppy image for node "
               "%(node)s", {'node': task.node.uuid})
@@ -280,10 +293,79 @@ def cleanup_floppy_image(task):
 
     :param task: an ironic node object.
     """
-    floppy_object_name = _get_floppy_image_name(task.node)
+    ImageHandler.unpublish_image_for_node(task.node, prefix='image')
+
+
+def prepare_configdrive_image(task, content):
+    """Prepare an image with configdrive.
+
+    :param task: a TaskManager instance containing the node to act on.
+    :param content: Config drive as a base64-encoded string.
+    :raises: ImageCreationFailed, if it failed while creating the image.
+    :raises: SwiftOperationError, if any operation with Swift fails.
+    :returns: image URL for the image.
+    """
+    # FIXME(dtantsur): download and convert?
+    if '://' in content:
+        raise exception.ImageCreationFailed(
+            _('URLs are not supported for configdrive images yet'))
+
+    with tempfile.TemporaryFile(dir=CONF.tempdir) as comp_tmpfile_obj:
+        comp_tmpfile_obj.write(base64.b64decode(content))
+        comp_tmpfile_obj.seek(0)
+        gz = gzip.GzipFile(fileobj=comp_tmpfile_obj, mode='rb')
+        with tempfile.NamedTemporaryFile(
+                dir=CONF.tempdir, suffix='.img') as image_tmpfile_obj:
+            shutil.copyfileobj(gz, image_tmpfile_obj)
+            image_tmpfile_obj.flush()
+            return prepare_disk_image(task, image_tmpfile_obj.name,
+                                      prefix='configdrive')
+
+
+def prepare_disk_image(task, content, prefix=None):
+    """Prepare an image with the given content.
+
+    If content is already an HTTP URL, return it unchanged.
+
+    :param task: a TaskManager instance containing the node to act on.
+    :param content: Content as a string with a file name or bytes with
+        contents.
+    :param prefix: Prefix to use for the object name.
+    :raises: ImageCreationFailed, if it failed while creating the image.
+    :raises: SwiftOperationError, if any operation with Swift fails.
+    :returns: image URL for the image.
+    """
+    object_name = _get_name(task.node, prefix=prefix)
+
+    LOG.debug("Creating a disk image for node %s", task.node.uuid)
 
     img_handler = ImageHandler(task.node.driver)
-    img_handler.unpublish_image(floppy_object_name)
+    if isinstance(content, str):
+        image_url = img_handler.publish_image(content, object_name)
+    else:
+        with tempfile.NamedTemporaryFile(
+                dir=CONF.tempdir, suffix='.img') as image_tmpfile_obj:
+            image_tmpfile_obj.write(content)
+            image_tmpfile_obj.flush()
+
+            image_tmpfile = image_tmpfile_obj.name
+            image_url = img_handler.publish_image(image_tmpfile, object_name)
+
+    LOG.debug("Created a disk image %(name)s for node %(node)s, "
+              "exposed as URL %(url)s", {'node': task.node.uuid,
+                                         'name': object_name,
+                                         'url': image_url})
+
+    return image_url
+
+
+def cleanup_disk_image(task, prefix=None):
+    """Deletes the image if it was created for the node.
+
+    :param task: an ironic node object.
+    :param prefix: Prefix to use for the object name.
+    """
+    ImageHandler.unpublish_image_for_node(task.node, prefix=prefix)
 
 
 def _prepare_iso_image(task, kernel_href, ramdisk_href,
@@ -366,7 +448,7 @@ def _prepare_iso_image(task, kernel_href, ramdisk_href,
             base_iso=base_iso,
             inject_files=inject_files)
 
-        iso_object_name = _get_iso_image_name(task.node)
+        iso_object_name = _get_name(task.node, prefix='boot', suffix='.iso')
 
         image_url = img_handler.publish_image(
             boot_iso_tmp_file, iso_object_name)

ironic/drivers/modules/redfish/boot.py

@@ -532,10 +532,21 @@ class RedfishVirtualMediaBoot(base.BootInterface):
             params.update(root_uuid=root_uuid)
 
         deploy_info = _parse_deploy_info(node)
+        configdrive = node.instance_info.get('configdrive')
         iso_ref = image_utils.prepare_boot_iso(task, deploy_info, **params)
         eject_vmedia(task, sushy.VIRTUAL_MEDIA_CD)
         _insert_vmedia(task, iso_ref, sushy.VIRTUAL_MEDIA_CD)
 
+        if configdrive and boot_option == 'ramdisk':
+            eject_vmedia(task, sushy.VIRTUAL_MEDIA_USBSTICK)
+            cd_ref = image_utils.prepare_configdrive_image(task, configdrive)
+            try:
+                _insert_vmedia(task, cd_ref, sushy.VIRTUAL_MEDIA_USBSTICK)
+            except exception.InvalidParameterValue:
+                raise exception.InstanceDeployFailure(
+                    _('Cannot attach configdrive for node %s: no suitable '
+                      'virtual USB slot has been found') % node.uuid)
+
         boot_mode_utils.sync_boot_mode(task)
 
         self._set_boot_device(task, boot_devices.CDROM, persistent=True)
@@ -562,6 +573,12 @@ class RedfishVirtualMediaBoot(base.BootInterface):
         if config_via_floppy:
             eject_vmedia(task, sushy.VIRTUAL_MEDIA_FLOPPY)
 
+        boot_option = deploy_utils.get_boot_option(task.node)
+        if (boot_option == 'ramdisk'
+                and task.node.instance_info.get('configdrive')):
+            eject_vmedia(task, sushy.VIRTUAL_MEDIA_USBSTICK)
+            image_utils.cleanup_disk_image(task, prefix='configdrive')
+
         image_utils.cleanup_iso_image(task)
 
     @classmethod

ironic/tests/unit/drivers/modules/redfish/test_boot.py

@@ -568,6 +568,7 @@ class RedfishVirtualMediaBootTestCase(db_base.DbTestCase):
 
     @mock.patch.object(redfish_boot.RedfishVirtualMediaBoot,
                        'clean_up_instance', autospec=True)
+    @mock.patch.object(image_utils, 'prepare_configdrive_image', autospec=True)
     @mock.patch.object(image_utils, 'prepare_boot_iso', autospec=True)
     @mock.patch.object(redfish_boot, 'eject_vmedia', autospec=True)
     @mock.patch.object(redfish_boot, '_insert_vmedia', autospec=True)
@@ -578,13 +579,14 @@ class RedfishVirtualMediaBootTestCase(db_base.DbTestCase):
     def test_prepare_instance_ramdisk_boot(
             self, mock_boot_mode_utils, mock_deploy_utils, mock_manager_utils,
             mock__parse_deploy_info, mock__insert_vmedia, mock__eject_vmedia,
-            mock_prepare_boot_iso, mock_clean_up_instance):
+            mock_prepare_boot_iso, mock_prepare_disk, mock_clean_up_instance):
-
+        configdrive = 'Y29udGVudA=='
         with task_manager.acquire(self.context, self.node.uuid,
                                   shared=True) as task:
             task.node.provision_state = states.DEPLOYING
             task.node.driver_internal_info[
                 'root_uuid_or_disk_id'] = self.node.uuid
+            task.node.instance_info['configdrive'] = configdrive
 
             mock_deploy_utils.get_boot_option.return_value = 'ramdisk'
 
@@ -596,18 +598,24 @@ class RedfishVirtualMediaBootTestCase(db_base.DbTestCase):
             mock__parse_deploy_info.return_value = d_info
 
             mock_prepare_boot_iso.return_value = 'image-url'
+            mock_prepare_disk.return_value = 'cd-url'
 
             task.driver.boot.prepare_instance(task)
 
             mock_clean_up_instance.assert_called_once_with(mock.ANY, task)
 
             mock_prepare_boot_iso.assert_called_once_with(task, d_info)
+            mock_prepare_disk.assert_called_once_with(task, configdrive)
 
-            mock__eject_vmedia.assert_called_once_with(
+            mock__eject_vmedia.assert_has_calls([
-                task, sushy.VIRTUAL_MEDIA_CD)
+                mock.call(task, sushy.VIRTUAL_MEDIA_CD),
+                mock.call(task, sushy.VIRTUAL_MEDIA_USBSTICK),
+            ])
 
-            mock__insert_vmedia.assert_called_once_with(
+            mock__insert_vmedia.assert_has_calls([
-                task, 'image-url', sushy.VIRTUAL_MEDIA_CD)
+                mock.call(task, 'image-url', sushy.VIRTUAL_MEDIA_CD),
+                mock.call(task, 'cd-url', sushy.VIRTUAL_MEDIA_USBSTICK),
+            ])
 
             mock_manager_utils.node_set_boot_device.assert_called_once_with(
                 task, boot_devices.CDROM, persistent=True)
@@ -633,6 +641,7 @@ class RedfishVirtualMediaBootTestCase(db_base.DbTestCase):
             task.node.provision_state = states.DEPLOYING
             task.node.driver_internal_info[
                 'root_uuid_or_disk_id'] = self.node.uuid
+            task.node.instance_info['configdrive'] = None
 
             mock_deploy_utils.get_boot_option.return_value = 'ramdisk'
 
@@ -679,6 +688,7 @@ class RedfishVirtualMediaBootTestCase(db_base.DbTestCase):
             task.node.provision_state = states.DEPLOYING
             i_info = task.node.instance_info
             i_info['boot_iso'] = "super-magic"
+            del i_info['configdrive']
             task.node.instance_info = i_info
             mock_deploy_utils.get_boot_option.return_value = 'ramdisk'
             mock__parse_deploy_info.return_value = {}
@@ -760,6 +770,29 @@ class RedfishVirtualMediaBootTestCase(db_base.DbTestCase):
         self.node.save()
         self._test_clean_up_instance()
 
+    @mock.patch.object(deploy_utils, 'get_boot_option', autospec=True)
+    @mock.patch.object(redfish_boot, 'eject_vmedia', autospec=True)
+    @mock.patch.object(image_utils, 'cleanup_disk_image', autospec=True)
+    @mock.patch.object(image_utils, 'cleanup_iso_image', autospec=True)
+    def test_clean_up_instance_ramdisk(self, mock_cleanup_iso_image,
+                                       mock_cleanup_disk_image,
+                                       mock__eject_vmedia,
+                                       mock_get_boot_option):
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            mock_get_boot_option.return_value = 'ramdisk'
+
+            task.driver.boot.clean_up_instance(task)
+
+            mock_cleanup_iso_image.assert_called_once_with(task)
+            mock_cleanup_disk_image.assert_called_once_with(
+                task, prefix='configdrive')
+            eject_calls = [mock.call(task, sushy.VIRTUAL_MEDIA_CD),
+                           mock.call(task, sushy.VIRTUAL_MEDIA_USBSTICK)]
+
+            mock__eject_vmedia.assert_has_calls(eject_calls)
+
     @mock.patch.object(redfish_boot, 'redfish_utils', autospec=True)
     def test__insert_vmedia_anew(self, mock_redfish_utils):
 

ironic/tests/unit/drivers/modules/test_image_utils.py

@@ -217,6 +217,93 @@ class RedfishImageUtilsTestCase(db_base.DbTestCase):
 
             self.assertEqual(expected_url, url)
 
+    @mock.patch.object(image_utils.ImageHandler, 'publish_image',
+                       autospec=True)
+    def test_prepare_disk_image(self, mock_publish_image):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            expected_url = 'https://a.b/c.f?e=f'
+            expected_object_name = task.node.uuid
+
+            def _publish(img_handler, tmp_file, object_name):
+                self.assertEqual(expected_object_name, object_name)
+                self.assertEqual(b'content', open(tmp_file, 'rb').read())
+                return expected_url
+
+            mock_publish_image.side_effect = _publish
+
+            url = image_utils.prepare_disk_image(task, b'content')
+
+            mock_publish_image.assert_called_once_with(mock.ANY, mock.ANY,
+                                                       expected_object_name)
+
+            self.assertEqual(expected_url, url)
+
+    @mock.patch.object(image_utils.ImageHandler, 'publish_image',
+                       autospec=True)
+    def test_prepare_disk_image_prefix(self, mock_publish_image):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            expected_url = 'https://a.b/c.f?e=f'
+            expected_object_name = 'configdrive-%s' % task.node.uuid
+
+            def _publish(img_handler, tmp_file, object_name):
+                self.assertEqual(expected_object_name, object_name)
+                self.assertEqual(b'content', open(tmp_file, 'rb').read())
+                return expected_url
+
+            mock_publish_image.side_effect = _publish
+
+            url = image_utils.prepare_disk_image(task, b'content',
+                                                 prefix='configdrive')
+
+            mock_publish_image.assert_called_once_with(mock.ANY, mock.ANY,
+                                                       expected_object_name)
+
+            self.assertEqual(expected_url, url)
+
+    @mock.patch.object(image_utils.ImageHandler, 'publish_image',
+                       autospec=True)
+    def test_prepare_disk_image_file(self, mock_publish_image):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            expected_url = 'https://a.b/c.f?e=f'
+            expected_object_name = task.node.uuid
+
+            def _publish(img_handler, tmp_file, object_name):
+                self.assertEqual(expected_object_name, object_name)
+                self.assertEqual(b'content', open(tmp_file, 'rb').read())
+                return expected_url
+
+            mock_publish_image.side_effect = _publish
+
+            with tempfile.NamedTemporaryFile() as fp:
+                fp.write(b'content')
+                fp.flush()
+                url = image_utils.prepare_disk_image(task, fp.name)
+
+            mock_publish_image.assert_called_once_with(mock.ANY, mock.ANY,
+                                                       expected_object_name)
+
+            self.assertEqual(expected_url, url)
+
+    @mock.patch.object(image_utils, 'prepare_disk_image', autospec=True)
+    def test_prepare_configdrive_image(self, mock_prepare):
+        expected_url = 'https://a.b/c.f?e=f'
+        encoded = 'H4sIAPJ8418C/0vOzytJzSsBAKkwxf4HAAAA'
+
+        def _prepare(task, content, prefix):
+            with open(content, 'rb') as fp:
+                self.assertEqual(b'content', fp.read())
+            return expected_url
+
+        mock_prepare.side_effect = _prepare
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            result = image_utils.prepare_configdrive_image(task, encoded)
+            self.assertEqual(expected_url, result)
+
     @mock.patch.object(image_utils.ImageHandler, 'unpublish_image',
                        autospec=True)
     def test_cleanup_iso_image(self, mock_unpublish):

ironic/tests/unit/drivers/third_party_driver_mock_specs.py

@@ -155,6 +155,7 @@ SUSHY_SPEC = (
     'STATE_ABSENT',
     'VIRTUAL_MEDIA_CD',
     'VIRTUAL_MEDIA_FLOPPY',
+    'VIRTUAL_MEDIA_USBSTICK',
     'APPLY_TIME_ON_RESET',
     'TASK_STATE_COMPLETED',
     'HEALTH_OK',

ironic/tests/unit/drivers/third_party_driver_mocks.py

@@ -217,6 +217,7 @@ if not sushy:
         STATE_ABSENT='absent',
         VIRTUAL_MEDIA_CD='cd',
         VIRTUAL_MEDIA_FLOPPY='floppy',
+        VIRTUAL_MEDIA_USBSTICK='usb',
         APPLY_TIME_ON_RESET='on reset',
         TASK_STATE_COMPLETED='completed',
         HEALTH_OK='ok',

releasenotes/notes/ramdisk-configdrive-142149339dd00b47.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Supports attaching configdrives when doing ``ramdisk`` deploy with the
+    ``redfish-virtual-media`` boot. A configdrive is attached to a free USB
+    slot. Swift must not be used for configdrive storage (this limitation will
+    be fixed later).