Add support for creating vfat and iso images

This commit introduces support for creating vfat images
and iso images in ironic. Vfat images will be used for
passing information to baremetal node using virtual
media through OOB channel. Iso images will be used for
booting up baremetal nodes via virtual media.

Change-Id: I42b88da02090aec310f567642d3174d0f482b585
Implements: blueprint ironic-ilo-virtualmedia-driver

etc/ironic/ironic.conf.sample

@@ -251,6 +251,13 @@
 # Force backing images to raw format. (boolean value)
 #force_raw_images=true
 
+# Path to isolinux binary file. (string value)
+#isolinux_bin=/usr/lib/syslinux/isolinux.bin
+
+# Template file for isolinux configuration file. (string
+# value)
+#isolinux_config_template=$pybasedir/common/isolinux_config.template
+
 
 #
 # Options defined in ironic.common.paths

etc/ironic/rootwrap.d/ironic-utils.filters

@@ -4,13 +4,15 @@
 [Filters]
 # ironic/drivers/modules/deploy_utils.py
 iscsiadm: CommandFilter, iscsiadm, root
-dd: CommandFilter, dd, root
 blkid: CommandFilter, blkid, root
 blockdev: CommandFilter, blockdev, root
 
 # ironic/common/utils.py
 mkswap: CommandFilter, mkswap, root
 mkfs: CommandFilter, mkfs, root
+mount: CommandFilter, mount, root
+umount: CommandFilter, umount, root
+dd: CommandFilter, dd, root
 
 # ironic/common/disk_partitioner.py
 fuser: CommandFilter, fuser, root

ironic/common/exception.py

@@ -437,3 +437,7 @@ class InsufficientDiskSpace(IronicException):
     message = _("Disk volume where '%(path)s' is located doesn't have "
                 "enough disk space. Required %(required)d MiB, "
                 "only %(actual)d MiB available space present.")
+
+
+class ImageCreationFailed(IronicException):
+    message = _('Creating %(image_type)s image failed: %(error)s')

ironic/common/images.py

@@ -20,28 +20,202 @@ Handling of VM disk images.
 """
 
 import os
+import shutil
 
+import jinja2
 from oslo.config import cfg
 
 from ironic.common import exception
+from ironic.common import i18n
 from ironic.common import image_service as service
+from ironic.common import paths
 from ironic.common import utils
 from ironic.openstack.common import fileutils
 from ironic.openstack.common import imageutils
 from ironic.openstack.common import log as logging
+from ironic.openstack.common import processutils
 
 LOG = logging.getLogger(__name__)
 
+_LE = i18n._LE
+
 image_opts = [
     cfg.BoolOpt('force_raw_images',
                 default=True,
                 help='Force backing images to raw format.'),
+    cfg.StrOpt('isolinux_bin',
+                default='/usr/lib/syslinux/isolinux.bin',
+                help='Path to isolinux binary file.'),
+    cfg.StrOpt('isolinux_config_template',
+                default=paths.basedir_def('common/isolinux_config.template'),
+                help='Template file for isolinux configuration file.'),
 ]
 
 CONF = cfg.CONF
 CONF.register_opts(image_opts)
 
 
+def _create_root_fs(root_directory, files_info):
+    """Creates a filesystem root in given directory.
+
+    Given a mapping of absolute path of files to their relative paths
+    within the filesystem, this method copies the files to their
+    destination.
+
+    :param root_directory: the filesystem root directory.
+    :param files_info: A dict containing absolute path of file to be copied
+        -> relative path within the vfat image. For example,
+        {
+         '/absolute/path/to/file' -> 'relative/path/within/root'
+         ...
+        }
+    :raises: OSError, if creation of any directory failed.
+    :raises: IOError, if copying any of the files failed.
+    """
+    for src_file, path in files_info.items():
+        target_file = os.path.join(root_directory, path)
+        dirname = os.path.dirname(target_file)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+        shutil.copyfile(src_file, target_file)
+
+
+def create_vfat_image(output_file, files_info=None, parameters=None,
+                      parameters_file='parameters.txt', fs_size_kib=100):
+    """Creates the fat fs image on the desired file.
+
+    This method copies the given files to a root directory (optional),
+    writes the parameters specified to the parameters file within the
+    root directory (optional), and then creates a vfat image of the root
+    directory.
+
+    :param output_file: The path to the file where the fat fs image needs
+        to be created.
+    :param files_info: A dict containing absolute path of file to be copied
+        -> relative path within the vfat image. For example,
+        {
+         '/absolute/path/to/file' -> 'relative/path/within/root'
+         ...
+        }
+    :param parameters: A dict containing key-value pairs of parameters.
+    :param parameters_file: The filename for the parameters file.
+    :param fs_size_kib: size of the vfat filesystem in KiB.
+    :raises: ImageCreationFailed, if image creation failed while doing any
+        of filesystem manipulation activities like creating dirs, mounting,
+        creating filesystem, copying files, etc.
+    """
+    try:
+        utils.dd('/dev/zero', output_file, 'count=1', "bs=%dKiB" % fs_size_kib)
+    except processutils.ProcessExecutionError as e:
+        raise exception.ImageCreationFailed(image_type='vfat', error=e)
+
+    with utils.tempdir() as tmpdir:
+
+        try:
+            utils.mkfs('vfat', output_file)
+            utils.mount(output_file, tmpdir, '-o', 'umask=0')
+        except processutils.ProcessExecutionError as e:
+            raise exception.ImageCreationFailed(image_type='vfat', error=e)
+
+        try:
+            if files_info:
+                _create_root_fs(tmpdir, files_info)
+
+            if parameters:
+                parameters_file = os.path.join(tmpdir, parameters_file)
+                params_list = ['%(key)s=%(val)s' % {'key': k, 'val': v}
+                           for k, v in parameters.items()]
+                file_contents = '\n'.join(params_list)
+                utils.write_to_file(parameters_file, file_contents)
+
+        except Exception as e:
+            LOG.exception(_LE("vfat image creation failed. Error: %s"), e)
+            raise exception.ImageCreationFailed(image_type='vfat', error=e)
+
+        finally:
+            try:
+                utils.umount(tmpdir)
+            except processutils.ProcessExecutionError as e:
+                raise exception.ImageCreationFailed(image_type='vfat', error=e)
+
+
+def _generate_isolinux_cfg(kernel_params):
+    """Generates a isolinux configuration file.
+
+    Given a given a list of strings containing kernel parameters, this method
+    returns the kernel cmdline string.
+    :param kernel_params: a list of strings(each element being a string like
+        'K=V' or 'K' or combination of them like 'K1=V1 K2 K3=V3') to be added
+        as the kernel cmdline.
+    :returns: a string containing the contents of the isolinux configuration
+        file.
+    """
+    if not kernel_params:
+        kernel_params = []
+    kernel_params_str = ' '.join(kernel_params)
+
+    template = CONF.isolinux_config_template
+    tmpl_path, tmpl_file = os.path.split(template)
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader(tmpl_path))
+    template = env.get_template(tmpl_file)
+
+    options = {'kernel': '/vmlinuz', 'ramdisk': '/initrd',
+               'kernel_params': kernel_params_str}
+
+    cfg = template.render(options)
+    return cfg
+
+
+def create_isolinux_image(output_file, kernel, ramdisk, kernel_params=None):
+    """Creates an isolinux image on the specified file.
+
+    Copies the provided kernel, ramdisk to a directory, generates the isolinux
+    configuration file using the kernel parameters provided, and then generates
+    a bootable ISO image.
+
+    :param output_file: the path to the file where the iso image needs to be
+        created.
+    :param kernel: the kernel to use.
+    :param ramdisk: the ramdisk to use.
+    :param kernel_params: a list of strings(each element being a string like
+        'K=V' or 'K' or combination of them like 'K1=V1,K2,...') to be added
+        as the kernel cmdline.
+    :raises: ImageCreationFailed, if image creation failed while copying files
+        or while running command to generate iso.
+    """
+    ISOLINUX_BIN = 'isolinux/isolinux.bin'
+    ISOLINUX_CFG = 'isolinux/isolinux.cfg'
+
+    with utils.tempdir() as tmpdir:
+
+        files_info = {
+                      kernel: 'vmlinuz',
+                      ramdisk: 'initrd',
+                      CONF.isolinux_bin: ISOLINUX_BIN,
+                     }
+
+        try:
+            _create_root_fs(tmpdir, files_info)
+        except (OSError, IOError) as e:
+            LOG.exception(_LE("Creating the filesystem root failed."))
+            raise exception.ImageCreationFailed(image_type='iso', error=e)
+
+        cfg = _generate_isolinux_cfg(kernel_params)
+
+        isolinux_cfg = os.path.join(tmpdir, ISOLINUX_CFG)
+        utils.write_to_file(isolinux_cfg, cfg)
+
+        try:
+            utils.execute('mkisofs', '-r', '-V', "BOOT IMAGE",
+                          '-cache-inodes', '-J', '-l', '-no-emul-boot',
+                          '-boot-load-size', '4', '-boot-info-table',
+                          '-b', ISOLINUX_BIN, '-o', output_file, tmpdir)
+        except processutils.ProcessExecutionError as e:
+            LOG.exception(_LE("Creating ISO image failed."))
+            raise exception.ImageCreationFailed(image_type='iso', error=e)
+
+
 def qemu_img_info(path):
     """Return an object containing the parsed output from qemu-img info."""
     if not os.path.exists(path):

ironic/common/isolinux_config.template

@@ -0,0 +1,5 @@
+default boot
+
+label boot
+kernel {{ kernel }}
+append initrd={{ ramdisk }} text {{ kernel_params }} --

ironic/common/utils.py

@@ -504,3 +504,44 @@ def is_uuid_like(val):
         return str(uuid.UUID(val)) == val
     except (TypeError, ValueError, AttributeError):
         return False
+
+
+def mount(src, dest, *args):
+    """Mounts a device/image file on specified location.
+
+    :param src: the path to the source file for mounting
+    :param dest: the path where it needs to be mounted.
+    :param args: a tuple containing the arguments to be
+        passed to mount command.
+    :raises: processutils.ProcessExecutionError if it failed
+        to run the process.
+    """
+    args = ('mount', ) + args + (src, dest)
+    execute(*args, run_as_root=True, check_exit_code=[0])
+
+
+def umount(loc, *args):
+    """Umounts a mounted location.
+
+    :param loc: the path to be unmounted.
+    :param args: a tuple containing the argumnets to be
+        passed to the umount command.
+    :raises: processutils.ProcessExecutionError if it failed
+        to run the process.
+    """
+    args = ('umount', ) + args + (loc, )
+    execute(*args, run_as_root=True, check_exit_code=[0])
+
+
+def dd(src, dst, *args):
+    """Execute dd from src to dst.
+
+    :param src: the input file for dd command.
+    :param dst: the output file for dd command.
+    :param args: a tuple containing the arguments to be
+        passed to dd command.
+    :raises: processutils.ProcessExecutionError if it failed
+        to run the process.
+    """
+    execute('dd', 'if=%s' % src, 'of=%s' % dst, *args,
+            run_as_root=True, check_exit_code=[0])

ironic/drivers/modules/deploy_utils.py

@@ -139,13 +139,7 @@ def is_block_device(dev):
 
 def dd(src, dst):
     """Execute dd from src to dst."""
-    utils.execute('dd',
-                  'if=%s' % src,
-                  'of=%s' % dst,
-                  'bs=1M',
-                  'oflag=direct',
-                  run_as_root=True,
-                  check_exit_code=[0])
+    utils.dd(src, dst, 'bs=1M', 'oflag=direct')
 
 
 def mkswap(dev, label='swap1'):

ironic/drivers/modules/iscsi_deploy.py

@@ -354,7 +354,8 @@ def validate_glance_image_properties(ctx, deploy_info, properties):
     :param ctx: security context
     :param deploy_info: the deploy_info to be validated
     :param properties: the list of image meta-properties to be validated.
-    :raises: InvalidParameterValue if the glance image doesn't exist.
+    :raises: InvalidParameterValue if connection to glance failed or
+        authorization for accessing image failed or if image doesn't exist.
     :raises: MissingParameterValue if the glance image doesn't contain
         the mentioned properties.
     """

ironic/tests/test_images.py

@@ -18,13 +18,21 @@
 
 import contextlib
 import fixtures
+import mock
+import os
+import shutil
 
+from oslo.config import cfg
 from oslo.utils import excutils
 
 from ironic.common import exception
 from ironic.common import images
+from ironic.common import utils
+from ironic.openstack.common import processutils
 from ironic.tests import base
 
+CONF = cfg.CONF
+
 
 class IronicImagesTestCase(base.TestCase):
     def test_fetch_raw_image(self):
@@ -111,3 +119,212 @@ class IronicImagesTestCase(base.TestCase):
         self.assertEqual(expected_commands, self.executes)
 
         del self.executes
+
+
+class FsImageTestCase(base.TestCase):
+
+    @mock.patch.object(shutil, 'copyfile')
+    @mock.patch.object(os, 'makedirs')
+    @mock.patch.object(os.path, 'dirname')
+    @mock.patch.object(os.path, 'exists')
+    def test__create_root_fs(self, path_exists_mock,
+                            dirname_mock, mkdir_mock, cp_mock):
+
+        path_exists_mock_func = lambda path: path == 'root_dir'
+
+        files_info = {
+                'a1': 'b1',
+                'a2': 'b2',
+                'a3': 'sub_dir/b3'}
+
+        path_exists_mock.side_effect = path_exists_mock_func
+        dirname_mock.side_effect = ['root_dir', 'root_dir',
+                                    'root_dir/sub_dir', 'root_dir/sub_dir']
+        images._create_root_fs('root_dir', files_info)
+        cp_mock.assert_any_call('a1', 'root_dir/b1')
+        cp_mock.assert_any_call('a2', 'root_dir/b2')
+        cp_mock.assert_any_call('a3', 'root_dir/sub_dir/b3')
+
+        path_exists_mock.assert_any_call('root_dir/sub_dir')
+        dirname_mock.assert_any_call('root_dir/b1')
+        dirname_mock.assert_any_call('root_dir/b2')
+        dirname_mock.assert_any_call('root_dir/sub_dir/b3')
+        mkdir_mock.assert_called_once_with('root_dir/sub_dir')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'write_to_file')
+    @mock.patch.object(utils, 'dd')
+    @mock.patch.object(utils, 'umount')
+    @mock.patch.object(utils, 'mount')
+    @mock.patch.object(utils, 'mkfs')
+    def test_create_vfat_image(self, mkfs_mock, mount_mock, umount_mock,
+            dd_mock, write_mock, tempdir_mock, create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tempdir'
+        tempdir_mock.return_value = mock_file_handle
+
+        parameters = {'p1': 'v1'}
+        files_info = {'a': 'b'}
+        images.create_vfat_image('tgt_file', parameters=parameters,
+                files_info=files_info, parameters_file='qwe',
+                fs_size_kib=1000)
+
+        dd_mock.assert_called_once_with('/dev/zero',
+                                         'tgt_file',
+                                         'count=1',
+                                         'bs=1000KiB')
+
+        mkfs_mock.assert_called_once_with('vfat', 'tgt_file')
+        mount_mock.assert_called_once_with('tgt_file', 'tempdir',
+                                           '-o', 'umask=0')
+
+        parameters_file_path = os.path.join('tempdir', 'qwe')
+        write_mock.assert_called_once_with(parameters_file_path, 'p1=v1')
+        create_root_fs_mock.assert_called_once_with('tempdir', files_info)
+        umount_mock.assert_called_once_with('tempdir')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'dd')
+    @mock.patch.object(utils, 'umount')
+    @mock.patch.object(utils, 'mount')
+    @mock.patch.object(utils, 'mkfs')
+    def test_create_vfat_image_always_umount(self, mkfs_mock, mount_mock,
+            umount_mock, dd_mock, tempdir_mock, create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tempdir'
+        tempdir_mock.return_value = mock_file_handle
+        files_info = {'a': 'b'}
+        create_root_fs_mock.side_effect = OSError()
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_vfat_image, 'tgt_file',
+                          files_info=files_info)
+
+        umount_mock.assert_called_once_with('tempdir')
+
+    @mock.patch.object(utils, 'dd')
+    def test_create_vfat_image_dd_fails(self, dd_mock):
+
+        dd_mock.side_effect = processutils.ProcessExecutionError
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_vfat_image, 'tgt_file')
+
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'dd')
+    @mock.patch.object(utils, 'mkfs')
+    def test_create_vfat_image_mkfs_fails(self, mkfs_mock, dd_mock,
+                                          tempdir_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tempdir'
+        tempdir_mock.return_value = mock_file_handle
+
+        mkfs_mock.side_effect = processutils.ProcessExecutionError
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_vfat_image, 'tgt_file')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'dd')
+    @mock.patch.object(utils, 'umount')
+    @mock.patch.object(utils, 'mount')
+    @mock.patch.object(utils, 'mkfs')
+    def test_create_vfat_image_umount_fails(self, mkfs_mock, mount_mock,
+            umount_mock, dd_mock, tempdir_mock, create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tempdir'
+        tempdir_mock.return_value = mock_file_handle
+        umount_mock.side_effect = processutils.ProcessExecutionError
+
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_vfat_image, 'tgt_file')
+
+    def test__generate_isolinux_cfg(self):
+
+        kernel_params = ['key1=value1', 'key2']
+        expected_cfg = ("default boot\n"
+                        "\n"
+                        "label boot\n"
+                        "kernel /vmlinuz\n"
+                        "append initrd=/initrd text key1=value1 key2 --")
+        cfg = images._generate_isolinux_cfg(kernel_params)
+        self.assertEqual(expected_cfg, cfg)
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'write_to_file')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'execute')
+    @mock.patch.object(images, '_generate_isolinux_cfg')
+    def test_create_isolinux_image(self, gen_cfg_mock, utils_mock,
+                                   tempdir_mock, write_to_file_mock,
+                                   create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tmpdir'
+        tempdir_mock.return_value = mock_file_handle
+
+        cfg = "cfg"
+        cfg_file = 'tmpdir/isolinux/isolinux.cfg'
+        gen_cfg_mock.return_value = cfg
+
+        params = ['a=b', 'c']
+
+        images.create_isolinux_image('tgt_file', 'path/to/kernel',
+                'path/to/ramdisk', kernel_params=params)
+
+        files_info = {
+                'path/to/kernel': 'vmlinuz',
+                'path/to/ramdisk': 'initrd',
+                CONF.isolinux_bin: 'isolinux/isolinux.bin'
+                }
+        create_root_fs_mock.assert_called_once_with('tmpdir', files_info)
+        gen_cfg_mock.assert_called_once_with(params)
+        write_to_file_mock.assert_called_once_with(cfg_file, cfg)
+
+        utils_mock.assert_called_once_with('mkisofs', '-r', '-V',
+                 "BOOT IMAGE", '-cache-inodes', '-J', '-l',
+                 '-no-emul-boot', '-boot-load-size',
+                 '4', '-boot-info-table', '-b', 'isolinux/isolinux.bin',
+                 '-o', 'tgt_file', 'tmpdir')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'execute')
+    def test_create_isolinux_image_rootfs_fails(self, utils_mock,
+                                                tempdir_mock,
+                                                create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tmpdir'
+        tempdir_mock.return_value = mock_file_handle
+        create_root_fs_mock.side_effect = IOError
+
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_isolinux_image,
+                          'tgt_file', 'path/to/kernel',
+                          'path/to/ramdisk')
+
+    @mock.patch.object(images, '_create_root_fs')
+    @mock.patch.object(utils, 'write_to_file')
+    @mock.patch.object(utils, 'tempdir')
+    @mock.patch.object(utils, 'execute')
+    @mock.patch.object(images, '_generate_isolinux_cfg')
+    def test_create_isolinux_image_mkisofs_fails(self, gen_cfg_mock,
+                                                 utils_mock,
+                                                 tempdir_mock,
+                                                 write_to_file_mock,
+                                                 create_root_fs_mock):
+
+        mock_file_handle = mock.MagicMock(spec=file)
+        mock_file_handle.__enter__.return_value = 'tmpdir'
+        tempdir_mock.return_value = mock_file_handle
+        utils_mock.side_effect = processutils.ProcessExecutionError
+
+        self.assertRaises(exception.ImageCreationFailed,
+                          images.create_isolinux_image,
+                          'tgt_file', 'path/to/kernel',
+                          'path/to/ramdisk')