diff --git a/ironic_python_agent/extensions/standby.py b/ironic_python_agent/extensions/standby.py index 496833a91..ab1414dfa 100644 --- a/ironic_python_agent/extensions/standby.py +++ b/ironic_python_agent/extensions/standby.py @@ -28,6 +28,7 @@ import requests from ironic_python_agent import errors from ironic_python_agent.extensions import base from ironic_python_agent import hardware +from ironic_python_agent import partition_utils from ironic_python_agent import utils CONF = cfg.CONF @@ -175,17 +176,17 @@ def _write_partition_image(image, image_info, device): raise errors.InvalidCommandParamsError(msg) try: - return disk_utils.work_on_disk(device, root_mb, - image_info['swap_mb'], - image_info['ephemeral_mb'], - image_info['ephemeral_format'], - image, node_uuid, - preserve_ephemeral=preserve_ep, - configdrive=configdrive, - boot_option=boot_option, - boot_mode=boot_mode, - disk_label=disk_label, - cpu_arch=cpu_arch) + return partition_utils.work_on_disk(device, root_mb, + image_info['swap_mb'], + image_info['ephemeral_mb'], + image_info['ephemeral_format'], + image, node_uuid, + preserve_ephemeral=preserve_ep, + configdrive=configdrive, + boot_option=boot_option, + boot_mode=boot_mode, + disk_label=disk_label, + cpu_arch=cpu_arch) except processutils.ProcessExecutionError as e: raise errors.ImageWriteError(device, e.exit_code, e.stdout, e.stderr) @@ -715,9 +716,9 @@ class StandbyExtension(base.BaseAgentExtension): # wherein new IPA is being used with older version # of Ironic that did not pass 'node_uuid' in 'image_info' node_uuid = image_info.get('node_uuid', 'local') - disk_utils.create_config_drive_partition(node_uuid, - device, - configdrive) + partition_utils.create_config_drive_partition(node_uuid, + device, + configdrive) self._fix_up_partition_uuids(image_info, device) msg = 'image ({}) written to device {} ' diff --git a/ironic_python_agent/partition_utils.py b/ironic_python_agent/partition_utils.py new file mode 100644 index 000000000..c32a994d0 --- /dev/null +++ b/ironic_python_agent/partition_utils.py @@ -0,0 +1,471 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Logic related to handling partitions. + +Imported from ironic-lib's disk_utils as of the following commit: +https://opendev.org/openstack/ironic-lib/commit/9fb5be348202f4854a455cd08f400ae12b99e1f2 +""" + +import base64 +import gzip +import io +import math +import os +import shutil +import tempfile + +from ironic_lib import disk_utils +from ironic_lib import exception +from ironic_lib import utils +from oslo_concurrency import processutils +from oslo_log import log +from oslo_utils import excutils +from oslo_utils import units +import requests + + +LOG = log.getLogger() + +MAX_CONFIG_DRIVE_SIZE_MB = 64 + +# Maximum disk size supported by MBR is 2TB (2 * 1024 * 1024 MB) +MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR = 2097152 + + +def get_configdrive(configdrive, node_uuid, tempdir=None): + """Get the information about size and location of the configdrive. + + :param configdrive: Base64 encoded Gzipped configdrive content or + configdrive HTTP URL. + :param node_uuid: Node's uuid. Used for logging. + :param tempdir: temporary directory for the temporary configdrive file + :raises: InstanceDeployFailure if it can't download or decode the + config drive. + :returns: A tuple with the size in MiB and path to the uncompressed + configdrive file. + + """ + # Check if the configdrive option is a HTTP URL or the content directly + is_url = utils.is_http_url(configdrive) + if is_url: + try: + data = requests.get(configdrive).content + except requests.exceptions.RequestException as e: + raise exception.InstanceDeployFailure( + "Can't download the configdrive content for node %(node)s " + "from '%(url)s'. Reason: %(reason)s" % + {'node': node_uuid, 'url': configdrive, 'reason': e}) + else: + data = configdrive + + configdrive_file = tempfile.NamedTemporaryFile(delete=False, + prefix='configdrive', + dir=tempdir) + + try: + data = io.BytesIO(base64.b64decode(data)) + except Exception as exc: + if isinstance(data, bytes): + LOG.debug('Config drive for node %(node)s is not base64 encoded ' + '(%(error)s), assuming binary', + {'node': node_uuid, 'error': exc}) + configdrive_mb = int(math.ceil(len(data) / units.Mi)) + configdrive_file.write(data) + configdrive_file.close() + return (configdrive_mb, configdrive_file.name) + else: + configdrive_file.close() + utils.unlink_without_raise(configdrive_file.name) + + error_msg = ('Config drive for node %(node)s is not base64 ' + 'encoded or the content is malformed. ' + '%(cls)s: %(err)s.' + % {'node': node_uuid, 'err': exc, + 'cls': type(exc).__name__}) + if is_url: + error_msg += ' Downloaded from "%s".' % configdrive + raise exception.InstanceDeployFailure(error_msg) + + configdrive_mb = 0 + with gzip.GzipFile('configdrive', 'rb', fileobj=data) as gunzipped: + try: + shutil.copyfileobj(gunzipped, configdrive_file) + except EnvironmentError as e: + # Delete the created file + utils.unlink_without_raise(configdrive_file.name) + raise exception.InstanceDeployFailure( + 'Encountered error while decompressing and writing ' + 'config drive for node %(node)s. Error: %(exc)s' % + {'node': node_uuid, 'exc': e}) + else: + # Get the file size and convert to MiB + configdrive_file.seek(0, os.SEEK_END) + bytes_ = configdrive_file.tell() + configdrive_mb = int(math.ceil(float(bytes_) / units.Mi)) + finally: + configdrive_file.close() + + return (configdrive_mb, configdrive_file.name) + + +def get_labelled_partition(device_path, label, node_uuid): + """Check and return if partition with given label exists + + :param device_path: The device path. + :param label: Partition label + :param node_uuid: UUID of the Node. Used for logging. + :raises: InstanceDeployFailure, if any disk partitioning related + commands fail. + :returns: block device file for partition if it exists; otherwise it + returns None. + """ + disk_utils.partprobe(device_path) + try: + output, err = utils.execute('lsblk', '-Po', 'name,label', device_path, + check_exit_code=[0, 1], + use_standard_locale=True, run_as_root=True) + + except (processutils.UnknownArgumentError, + processutils.ProcessExecutionError, OSError) as e: + msg = ('Failed to retrieve partition labels on disk %(disk)s ' + 'for node %(node)s. Error: %(error)s' % + {'disk': device_path, 'node': node_uuid, 'error': e}) + LOG.error(msg) + raise exception.InstanceDeployFailure(msg) + + found_part = None + if output: + for dev in utils.parse_device_tags(output): + if dev['LABEL'].upper() == label.upper(): + if found_part: + found_2 = '/dev/%(part)s' % {'part': dev['NAME'].strip()} + found = [found_part, found_2] + raise exception.InstanceDeployFailure( + 'More than one partition with label "%(label)s" ' + 'exists on device %(device)s for node %(node)s: ' + '%(found)s.' % + {'label': label, 'device': device_path, + 'node': node_uuid, 'found': ' and '.join(found)}) + found_part = '/dev/%(part)s' % {'part': dev['NAME'].strip()} + + return found_part + + +def work_on_disk(dev, root_mb, swap_mb, ephemeral_mb, ephemeral_format, + image_path, node_uuid, preserve_ephemeral=False, + configdrive=None, boot_option="netboot", boot_mode="bios", + tempdir=None, disk_label=None, cpu_arch="", conv_flags=None): + """Create partitions and copy an image to the root partition. + + :param dev: Path for the device to work on. + :param root_mb: Size of the root partition in megabytes. + :param swap_mb: Size of the swap partition in megabytes. + :param ephemeral_mb: Size of the ephemeral partition in megabytes. If 0, + no ephemeral partition will be created. + :param ephemeral_format: The type of file system to format the ephemeral + partition. + :param image_path: Path for the instance's disk image. If ``None``, + the root partition is prepared but not populated. + :param node_uuid: node's uuid. Used for logging. + :param preserve_ephemeral: If True, no filesystem is written to the + ephemeral block device, preserving whatever content it had (if the + partition table has not changed). + :param configdrive: Optional. Base64 encoded Gzipped configdrive content + or configdrive HTTP URL. + :param boot_option: Can be "local" or "netboot". "netboot" by default. + :param boot_mode: Can be "bios" or "uefi". "bios" by default. + :param tempdir: A temporary directory + :param disk_label: The disk label to be used when creating the + partition table. Valid values are: "msdos", "gpt" or None; If None + Ironic will figure it out according to the boot_mode parameter. + :param cpu_arch: Architecture of the node the disk device belongs to. + When using the default value of None, no architecture specific + steps will be taken. This default should be used for x86_64. When + set to ppc64*, architecture specific steps are taken for booting a + partition image locally. + :param conv_flags: Flags that need to be sent to the dd command, to control + the conversion of the original file when copying to the host. It can + contain several options separated by commas. + :returns: a dictionary containing the following keys: + 'root uuid': UUID of root partition + 'efi system partition uuid': UUID of the uefi system partition + (if boot mode is uefi). + `partitions`: mapping of partition types to their device paths. + NOTE: If key exists but value is None, it means partition doesn't + exist. + """ + # the only way for preserve_ephemeral to be set to true is if we are + # rebuilding an instance with --preserve_ephemeral. + commit = not preserve_ephemeral + # now if we are committing the changes to disk clean first. + if commit: + disk_utils.destroy_disk_metadata(dev, node_uuid) + + try: + # If requested, get the configdrive file and determine the size + # of the configdrive partition + configdrive_mb = 0 + configdrive_file = None + if configdrive: + configdrive_mb, configdrive_file = get_configdrive( + configdrive, node_uuid, tempdir=tempdir) + + part_dict = disk_utils.make_partitions(dev, + root_mb, swap_mb, ephemeral_mb, + configdrive_mb, node_uuid, + commit=commit, + boot_option=boot_option, + boot_mode=boot_mode, + disk_label=disk_label, + cpu_arch=cpu_arch) + LOG.info("Successfully completed the disk device" + " %(dev)s partitioning for node %(node)s", + {'dev': dev, "node": node_uuid}) + + ephemeral_part = part_dict.get('ephemeral') + swap_part = part_dict.get('swap') + configdrive_part = part_dict.get('configdrive') + root_part = part_dict.get('root') + + if not disk_utils.is_block_device(root_part): + raise exception.InstanceDeployFailure( + "Root device '%s' not found" % root_part) + + for part in ('swap', 'ephemeral', 'configdrive', + 'efi system partition', 'PReP Boot partition'): + part_device = part_dict.get(part) + LOG.debug("Checking for %(part)s device (%(dev)s) on node " + "%(node)s.", {'part': part, 'dev': part_device, + 'node': node_uuid}) + if part_device and not disk_utils.is_block_device(part_device): + raise exception.InstanceDeployFailure( + "'%(partition)s' device '%(part_device)s' not found" % + {'partition': part, 'part_device': part_device}) + + # If it's a uefi localboot, then we have created the efi system + # partition. Create a fat filesystem on it. + if boot_mode == "uefi" and boot_option == "local": + efi_system_part = part_dict.get('efi system partition') + utils.mkfs(fs='vfat', path=efi_system_part, label='efi-part') + + if configdrive_part: + # Copy the configdrive content to the configdrive partition + disk_utils.dd(configdrive_file, configdrive_part, + conv_flags=conv_flags) + LOG.info("Configdrive for node %(node)s successfully copied " + "onto partition %(partition)s", + {'node': node_uuid, 'partition': configdrive_part}) + + finally: + # If the configdrive was requested make sure we delete the file + # after copying the content to the partition + if configdrive_file: + utils.unlink_without_raise(configdrive_file) + + if image_path is not None: + disk_utils.populate_image(image_path, root_part, conv_flags=conv_flags) + LOG.info("Image for %(node)s successfully populated", + {'node': node_uuid}) + else: + LOG.debug("Root partition for %s was created, but not populated", + node_uuid) + + if swap_part: + utils.mkfs(fs='swap', path=swap_part, label='swap1') + LOG.info("Swap partition %(swap)s successfully formatted " + "for node %(node)s", + {'swap': swap_part, 'node': node_uuid}) + + if ephemeral_part and not preserve_ephemeral: + utils.mkfs(fs=ephemeral_format, path=ephemeral_part, + label="ephemeral0") + LOG.info("Ephemeral partition %(ephemeral)s successfully " + "formatted for node %(node)s", + {'ephemeral': ephemeral_part, 'node': node_uuid}) + + uuids_to_return = { + 'root uuid': root_part, + 'efi system partition uuid': part_dict.get('efi system partition'), + } + + if cpu_arch.startswith('ppc'): + uuids_to_return[ + 'PReP Boot partition uuid' + ] = part_dict.get('PReP Boot partition') + + try: + for part, part_dev in uuids_to_return.items(): + if part_dev: + uuids_to_return[part] = disk_utils.block_uuid(part_dev) + + except processutils.ProcessExecutionError: + with excutils.save_and_reraise_exception(): + LOG.error("Failed to detect %s", part) + + return dict(partitions=part_dict, **uuids_to_return) + + +def create_config_drive_partition(node_uuid, device, configdrive): + """Create a partition for config drive + + Checks if the device is GPT or MBR partitioned and creates config drive + partition accordingly. + + :param node_uuid: UUID of the Node. + :param device: The device path. + :param configdrive: Base64 encoded Gzipped configdrive content or + configdrive HTTP URL. + :raises: InstanceDeployFailure if config drive size exceeds maximum limit + or if it fails to create config drive. + """ + confdrive_file = None + try: + config_drive_part = get_labelled_partition( + device, disk_utils.CONFIGDRIVE_LABEL, node_uuid) + + confdrive_mb, confdrive_file = get_configdrive(configdrive, node_uuid) + if confdrive_mb > MAX_CONFIG_DRIVE_SIZE_MB: + raise exception.InstanceDeployFailure( + 'Config drive size exceeds maximum limit of 64MiB. ' + 'Size of the given config drive is %(size)d MiB for ' + 'node %(node)s.' + % {'size': confdrive_mb, 'node': node_uuid}) + + LOG.debug("Adding config drive partition %(size)d MiB to " + "device: %(dev)s for node %(node)s", + {'dev': device, 'size': confdrive_mb, 'node': node_uuid}) + + disk_utils.fix_gpt_partition(device, node_uuid) + if config_drive_part: + LOG.debug("Configdrive for node %(node)s exists at " + "%(part)s", + {'node': node_uuid, 'part': config_drive_part}) + else: + cur_parts = set(part['number'] + for part in disk_utils.list_partitions(device)) + + if disk_utils.get_partition_table_type(device) == 'gpt': + create_option = '0:-%dMB:0' % MAX_CONFIG_DRIVE_SIZE_MB + utils.execute('sgdisk', '-n', create_option, device, + run_as_root=True) + else: + # Check if the disk has 4 partitions. The MBR based disk + # cannot have more than 4 partitions. + # TODO(stendulker): One can use logical partitions to create + # a config drive if there are 3 primary partitions. + # https://bugs.launchpad.net/ironic/+bug/1561283 + try: + pp_count, lp_count = disk_utils.count_mbr_partitions( + device) + except ValueError as e: + raise exception.InstanceDeployFailure( + 'Failed to check the number of primary partitions ' + 'present on %(dev)s for node %(node)s. Error: ' + '%(error)s' % {'dev': device, 'node': node_uuid, + 'error': e}) + if pp_count > 3: + raise exception.InstanceDeployFailure( + 'Config drive cannot be created for node %(node)s. ' + 'Disk (%(dev)s) uses MBR partitioning and already ' + 'has %(parts)d primary partitions.' + % {'node': node_uuid, 'dev': device, + 'parts': pp_count}) + + # Check if disk size exceeds 2TB msdos limit + startlimit = '-%dMiB' % MAX_CONFIG_DRIVE_SIZE_MB + endlimit = '-0' + if _is_disk_larger_than_max_size(device, node_uuid): + # Need to create a small partition at 2TB limit + LOG.warning("Disk size is larger than 2TB for " + "node %(node)s. Creating config drive " + "at the end of the disk %(disk)s.", + {'node': node_uuid, 'disk': device}) + startlimit = (MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR + - MAX_CONFIG_DRIVE_SIZE_MB - 1) + endlimit = MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR - 1 + + utils.execute('parted', '-a', 'optimal', '-s', '--', device, + 'mkpart', 'primary', 'fat32', startlimit, + endlimit, run_as_root=True) + # Trigger device rescan + disk_utils.trigger_device_rescan(device) + + upd_parts = set(part['number'] + for part in disk_utils.list_partitions(device)) + new_part = set(upd_parts) - set(cur_parts) + if len(new_part) != 1: + raise exception.InstanceDeployFailure( + 'Disk partitioning failed on device %(device)s. ' + 'Unable to retrieve config drive partition information.' + % {'device': device}) + + config_drive_part = disk_utils.partition_index_to_path( + device, new_part.pop()) + + disk_utils.udev_settle() + + # NOTE(vsaienko): check that devise actually exists, + # it is not handled by udevadm when using ISCSI, for more info see: + # https://bugs.launchpad.net/ironic/+bug/1673731 + # Do not use 'udevadm settle --exit-if-exist' here + LOG.debug('Waiting for the config drive partition %(part)s ' + 'on node %(node)s to be ready for writing.', + {'part': config_drive_part, 'node': node_uuid}) + utils.execute('test', '-e', config_drive_part, attempts=15, + delay_on_retry=True) + + disk_utils.dd(confdrive_file, config_drive_part) + LOG.info("Configdrive for node %(node)s successfully " + "copied onto partition %(part)s", + {'node': node_uuid, 'part': config_drive_part}) + + except (processutils.UnknownArgumentError, + processutils.ProcessExecutionError, OSError) as e: + msg = ('Failed to create config drive on disk %(disk)s ' + 'for node %(node)s. Error: %(error)s' % + {'disk': device, 'node': node_uuid, 'error': e}) + LOG.error(msg) + raise exception.InstanceDeployFailure(msg) + finally: + # If the configdrive was requested make sure we delete the file + # after copying the content to the partition + if confdrive_file: + utils.unlink_without_raise(confdrive_file) + + +def _is_disk_larger_than_max_size(device, node_uuid): + """Check if total disk size exceeds 2TB msdos limit + + :param device: device path. + :param node_uuid: node's uuid. Used for logging. + :raises: InstanceDeployFailure, if any disk partitioning related + commands fail. + :returns: True if total disk size exceeds 2TB. Returns False otherwise. + """ + try: + disksize_bytes, err = utils.execute('blockdev', '--getsize64', + device, + use_standard_locale=True, + run_as_root=True) + except (processutils.UnknownArgumentError, + processutils.ProcessExecutionError, OSError) as e: + msg = ('Failed to get size of disk %(disk)s for node %(node)s. ' + 'Error: %(error)s' % + {'disk': device, 'node': node_uuid, 'error': e}) + LOG.error(msg) + raise exception.InstanceDeployFailure(msg) + + disksize_mb = int(disksize_bytes.strip()) // 1024 // 1024 + + return disksize_mb > MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR diff --git a/ironic_python_agent/tests/unit/extensions/test_standby.py b/ironic_python_agent/tests/unit/extensions/test_standby.py index 79b4a6ca4..1f9856527 100644 --- a/ironic_python_agent/tests/unit/extensions/test_standby.py +++ b/ironic_python_agent/tests/unit/extensions/test_standby.py @@ -24,6 +24,7 @@ import requests from ironic_python_agent import errors from ironic_python_agent.extensions import standby from ironic_python_agent import hardware +from ironic_python_agent import partition_utils from ironic_python_agent.tests.unit import base from ironic_python_agent import utils @@ -200,7 +201,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) - @mock.patch('ironic_lib.disk_utils.work_on_disk', autospec=True) + @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) def test_write_partition_image_exception(self, work_on_disk_mock, image_mb_mock, execute_mock, open_mock, @@ -247,7 +248,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) - @mock.patch('ironic_lib.disk_utils.work_on_disk', autospec=True) + @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) def test_write_partition_image_no_node_uuid(self, work_on_disk_mock, image_mb_mock, execute_mock, open_mock, @@ -297,7 +298,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) - @mock.patch('ironic_lib.disk_utils.work_on_disk', autospec=True) + @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) def test_write_partition_image_exception_image_mb(self, work_on_disk_mock, image_mb_mock, @@ -322,7 +323,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True) @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) - @mock.patch('ironic_lib.disk_utils.work_on_disk', autospec=True) + @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) def test_write_partition_image(self, image_mb_mock, work_on_disk_mock, execute_mock, open_mock, dispatch_mock): @@ -682,8 +683,8 @@ class TestStandbyExtension(base.IronicAgentTest): autospec=True) @mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True) - @mock.patch('ironic_lib.disk_utils.create_config_drive_partition', - autospec=True) + @mock.patch.object(partition_utils, 'create_config_drive_partition', + autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', autospec=True) @mock.patch('ironic_python_agent.extensions.standby._write_image', @@ -733,8 +734,8 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch('ironic_python_agent.utils.execute', autospec=True) @mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True) - @mock.patch('ironic_lib.disk_utils.create_config_drive_partition', - autospec=True) + @mock.patch.object(partition_utils, 'create_config_drive_partition', + autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', autospec=True) @mock.patch('ironic_python_agent.extensions.standby._write_image', @@ -804,8 +805,8 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch('ironic_lib.disk_utils.get_disk_identifier', lambda dev: 'ROOT') @mock.patch('ironic_python_agent.utils.execute', autospec=True) - @mock.patch('ironic_lib.disk_utils.create_config_drive_partition', - autospec=True) + @mock.patch.object(partition_utils, 'create_config_drive_partition', + autospec=True) @mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', @@ -848,9 +849,9 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch('ironic_lib.disk_utils.get_disk_identifier', lambda dev: 'ROOT') - @mock.patch('ironic_lib.disk_utils.work_on_disk', autospec=True) - @mock.patch('ironic_lib.disk_utils.create_config_drive_partition', - autospec=True) + @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) + @mock.patch.object(partition_utils, 'create_config_drive_partition', + autospec=True) @mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', @@ -899,8 +900,8 @@ class TestStandbyExtension(base.IronicAgentTest): autospec=True) @mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True) - @mock.patch('ironic_lib.disk_utils.create_config_drive_partition', - autospec=True) + @mock.patch.object(partition_utils, 'create_config_drive_partition', + autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', autospec=True) @mock.patch('ironic_python_agent.extensions.standby._write_image', @@ -952,9 +953,9 @@ class TestStandbyExtension(base.IronicAgentTest): lambda _dev: [mock.Mock()]) @mock.patch('ironic_lib.disk_utils.get_disk_identifier', lambda dev: 'ROOT') - @mock.patch('ironic_lib.disk_utils.work_on_disk', autospec=True) - @mock.patch('ironic_lib.disk_utils.create_config_drive_partition', - autospec=True) + @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) + @mock.patch.object(partition_utils, 'create_config_drive_partition', + autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', autospec=True) @mock.patch('ironic_python_agent.extensions.standby.StandbyExtension' @@ -1379,7 +1380,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) - @mock.patch('ironic_lib.disk_utils.work_on_disk', autospec=True) + @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) def test_write_partition_image_no_node_uuid_uefi( self, work_on_disk_mock, image_mb_mock, diff --git a/ironic_python_agent/tests/unit/test_partition_utils.py b/ironic_python_agent/tests/unit/test_partition_utils.py new file mode 100644 index 000000000..d3e1bc903 --- /dev/null +++ b/ironic_python_agent/tests/unit/test_partition_utils.py @@ -0,0 +1,1040 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import shutil +import tempfile +from unittest import mock + +from ironic_lib import disk_partitioner +from ironic_lib import disk_utils +from ironic_lib import exception +from ironic_lib import utils +from oslo_concurrency import processutils +import requests + +from ironic_python_agent import partition_utils +from ironic_python_agent.tests.unit import base + + +@mock.patch.object(shutil, 'copyfileobj', autospec=True) +@mock.patch.object(requests, 'get', autospec=True) +class GetConfigdriveTestCase(base.IronicAgentTest): + + @mock.patch.object(gzip, 'GzipFile', autospec=True) + def test_get_configdrive(self, mock_gzip, mock_requests, mock_copy): + mock_requests.return_value = mock.MagicMock(content='Zm9vYmFy') + tempdir = tempfile.mkdtemp() + (size, path) = partition_utils.get_configdrive('http://1.2.3.4/cd', + 'fake-node-uuid', + tempdir=tempdir) + self.assertTrue(path.startswith(tempdir)) + mock_requests.assert_called_once_with('http://1.2.3.4/cd') + mock_gzip.assert_called_once_with('configdrive', 'rb', + fileobj=mock.ANY) + mock_copy.assert_called_once_with(mock.ANY, mock.ANY) + + def test_get_configdrive_binary(self, mock_requests, mock_copy): + mock_requests.return_value = mock.MagicMock(content=b'content') + tempdir = tempfile.mkdtemp() + (size, path) = partition_utils.get_configdrive('http://1.2.3.4/cd', + 'fake-node-uuid', + tempdir=tempdir) + self.assertTrue(path.startswith(tempdir)) + self.assertEqual(b'content', open(path, 'rb').read()) + mock_requests.assert_called_once_with('http://1.2.3.4/cd') + self.assertFalse(mock_copy.called) + + @mock.patch.object(gzip, 'GzipFile', autospec=True) + def test_get_configdrive_base64_string(self, mock_gzip, mock_requests, + mock_copy): + partition_utils.get_configdrive('Zm9vYmFy', 'fake-node-uuid') + self.assertFalse(mock_requests.called) + mock_gzip.assert_called_once_with('configdrive', 'rb', + fileobj=mock.ANY) + mock_copy.assert_called_once_with(mock.ANY, mock.ANY) + + def test_get_configdrive_bad_url(self, mock_requests, mock_copy): + mock_requests.side_effect = requests.exceptions.RequestException + self.assertRaises(exception.InstanceDeployFailure, + partition_utils.get_configdrive, + 'http://1.2.3.4/cd', 'fake-node-uuid') + self.assertFalse(mock_copy.called) + + def test_get_configdrive_base64_error(self, mock_requests, mock_copy): + self.assertRaises(exception.InstanceDeployFailure, + partition_utils.get_configdrive, + 'malformed', 'fake-node-uuid') + self.assertFalse(mock_copy.called) + + @mock.patch.object(gzip, 'GzipFile', autospec=True) + def test_get_configdrive_gzip_error(self, mock_gzip, mock_requests, + mock_copy): + mock_requests.return_value = mock.MagicMock(content='Zm9vYmFy') + mock_copy.side_effect = IOError + self.assertRaises(exception.InstanceDeployFailure, + partition_utils.get_configdrive, + 'http://1.2.3.4/cd', 'fake-node-uuid') + mock_requests.assert_called_once_with('http://1.2.3.4/cd') + mock_gzip.assert_called_once_with('configdrive', 'rb', + fileobj=mock.ANY) + mock_copy.assert_called_once_with(mock.ANY, mock.ANY) + + +@mock.patch.object(utils, 'execute', autospec=True) +class GetLabelledPartitionTestCases(base.IronicAgentTest): + + def setUp(self): + super(GetLabelledPartitionTestCases, self).setUp() + self.dev = "/dev/fake" + self.config_part_label = "config-2" + self.node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + + def test_get_partition_present(self, mock_execute): + lsblk_output = 'NAME="fake12" LABEL="config-2"\n' + part_result = '/dev/fake12' + mock_execute.side_effect = [(None, ''), (lsblk_output, '')] + result = partition_utils.get_labelled_partition(self.dev, + self.config_part_label, + self.node_uuid) + self.assertEqual(part_result, result) + execute_calls = [ + mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('lsblk', '-Po', 'name,label', self.dev, + check_exit_code=[0, 1], + use_standard_locale=True, run_as_root=True) + ] + mock_execute.assert_has_calls(execute_calls) + + def test_get_partition_present_uppercase(self, mock_execute): + lsblk_output = 'NAME="fake12" LABEL="CONFIG-2"\n' + part_result = '/dev/fake12' + mock_execute.side_effect = [(None, ''), (lsblk_output, '')] + result = partition_utils.get_labelled_partition(self.dev, + self.config_part_label, + self.node_uuid) + self.assertEqual(part_result, result) + execute_calls = [ + mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('lsblk', '-Po', 'name,label', self.dev, + check_exit_code=[0, 1], + use_standard_locale=True, run_as_root=True) + ] + mock_execute.assert_has_calls(execute_calls) + + def test_get_partition_absent(self, mock_execute): + mock_execute.side_effect = [(None, ''), + (None, '')] + result = partition_utils.get_labelled_partition(self.dev, + self.config_part_label, + self.node_uuid) + self.assertIsNone(result) + execute_calls = [ + mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('lsblk', '-Po', 'name,label', self.dev, + check_exit_code=[0, 1], + use_standard_locale=True, run_as_root=True) + ] + mock_execute.assert_has_calls(execute_calls) + + def test_get_partition_DeployFail_exc(self, mock_execute): + label = 'config-2' + lsblk_output = ('NAME="fake12" LABEL="%s"\n' + 'NAME="fake13" LABEL="%s"\n' % + (label, label)) + mock_execute.side_effect = [(None, ''), (lsblk_output, '')] + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'fake .*fake12 .*fake13', + partition_utils.get_labelled_partition, + self.dev, self.config_part_label, + self.node_uuid) + execute_calls = [ + mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('lsblk', '-Po', 'name,label', self.dev, + check_exit_code=[0, 1], + use_standard_locale=True, run_as_root=True) + ] + mock_execute.assert_has_calls(execute_calls) + + @mock.patch.object(partition_utils.LOG, 'error', autospec=True) + def test_get_partition_exc(self, mock_log, mock_execute): + mock_execute.side_effect = processutils.ProcessExecutionError + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'Failed to retrieve partition labels', + partition_utils.get_labelled_partition, + self.dev, self.config_part_label, + self.node_uuid) + execute_calls = [ + mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('lsblk', '-Po', 'name,label', self.dev, + check_exit_code=[0, 1], + use_standard_locale=True, run_as_root=True) + ] + mock_execute.assert_has_calls(execute_calls) + self.assertEqual(1, mock_log.call_count) + + +@mock.patch.object(utils, 'execute', autospec=True) +class IsDiskLargerThanMaxSizeTestCases(base.IronicAgentTest): + + dev = "/dev/fake" + node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + + def _test_is_disk_larger_than_max_size(self, mock_execute, blk_out): + mock_execute.return_value = ('%s\n' % blk_out, '') + result = partition_utils._is_disk_larger_than_max_size(self.dev, + self.node_uuid) + mock_execute.assert_called_once_with('blockdev', '--getsize64', + '/dev/fake', run_as_root=True, + use_standard_locale=True) + return result + + def test_is_disk_larger_than_max_size_false(self, mock_execute): + blkid_out = "53687091200" + ret = self._test_is_disk_larger_than_max_size(mock_execute, + blk_out=blkid_out) + self.assertFalse(ret) + + def test_is_disk_larger_than_max_size_true(self, mock_execute): + blkid_out = "4398046511104" + ret = self._test_is_disk_larger_than_max_size(mock_execute, + blk_out=blkid_out) + self.assertTrue(ret) + + @mock.patch.object(partition_utils.LOG, 'error', autospec=True) + def test_is_disk_larger_than_max_size_exc(self, mock_log, mock_execute): + mock_execute.side_effect = processutils.ProcessExecutionError + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'Failed to get size of disk', + partition_utils._is_disk_larger_than_max_size, + self.dev, self.node_uuid) + mock_execute.assert_called_once_with('blockdev', '--getsize64', + '/dev/fake', run_as_root=True, + use_standard_locale=True) + self.assertEqual(1, mock_log.call_count) + + +@mock.patch.object(disk_partitioner.DiskPartitioner, 'commit', lambda _: None) +class WorkOnDiskTestCase(base.IronicAgentTest): + + def setUp(self): + super(WorkOnDiskTestCase, self).setUp() + self.image_path = '/tmp/xyz/image' + self.root_mb = 128 + self.swap_mb = 64 + self.ephemeral_mb = 0 + self.ephemeral_format = None + self.configdrive_mb = 0 + self.node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + self.dev = '/dev/fake' + self.swap_part = '/dev/fake-part1' + self.root_part = '/dev/fake-part2' + + self.mock_ibd_obj = mock.patch.object( + disk_utils, 'is_block_device', autospec=True) + self.mock_ibd = self.mock_ibd_obj.start() + self.addCleanup(self.mock_ibd_obj.stop) + self.mock_mp_obj = mock.patch.object( + disk_utils, 'make_partitions', autospec=True) + self.mock_mp = self.mock_mp_obj.start() + self.addCleanup(self.mock_mp_obj.stop) + self.mock_remlbl_obj = mock.patch.object( + disk_utils, 'destroy_disk_metadata', autospec=True) + self.mock_remlbl = self.mock_remlbl_obj.start() + self.addCleanup(self.mock_remlbl_obj.stop) + self.mock_mp.return_value = {'swap': self.swap_part, + 'root': self.root_part} + + def test_no_root_partition(self): + self.mock_ibd.return_value = False + self.assertRaises(exception.InstanceDeployFailure, + partition_utils.work_on_disk, self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.ephemeral_format, self.image_path, + self.node_uuid) + self.mock_ibd.assert_called_once_with(self.root_part) + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.configdrive_mb, + self.node_uuid, commit=True, + boot_option="netboot", + boot_mode="bios", + disk_label=None, + cpu_arch="") + + def test_no_swap_partition(self): + self.mock_ibd.side_effect = iter([True, False]) + calls = [mock.call(self.root_part), + mock.call(self.swap_part)] + self.assertRaises(exception.InstanceDeployFailure, + partition_utils.work_on_disk, self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.ephemeral_format, self.image_path, + self.node_uuid) + self.assertEqual(self.mock_ibd.call_args_list, calls) + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.configdrive_mb, + self.node_uuid, commit=True, + boot_option="netboot", + boot_mode="bios", + disk_label=None, + cpu_arch="") + + def test_no_ephemeral_partition(self): + ephemeral_part = '/dev/fake-part1' + swap_part = '/dev/fake-part2' + root_part = '/dev/fake-part3' + ephemeral_mb = 256 + ephemeral_format = 'exttest' + + self.mock_mp.return_value = {'ephemeral': ephemeral_part, + 'swap': swap_part, + 'root': root_part} + self.mock_ibd.side_effect = iter([True, True, False]) + calls = [mock.call(root_part), + mock.call(swap_part), + mock.call(ephemeral_part)] + self.assertRaises(exception.InstanceDeployFailure, + partition_utils.work_on_disk, self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, ephemeral_format, + self.image_path, self.node_uuid) + self.assertEqual(self.mock_ibd.call_args_list, calls) + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, + self.configdrive_mb, + self.node_uuid, commit=True, + boot_option="netboot", + boot_mode="bios", + disk_label=None, + cpu_arch="") + + @mock.patch.object(utils, 'unlink_without_raise', autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', autospec=True) + def test_no_configdrive_partition(self, mock_configdrive, mock_unlink): + mock_configdrive.return_value = (10, 'fake-path') + swap_part = '/dev/fake-part1' + configdrive_part = '/dev/fake-part2' + root_part = '/dev/fake-part3' + configdrive_url = 'http://1.2.3.4/cd' + configdrive_mb = 10 + + self.mock_mp.return_value = {'swap': swap_part, + 'configdrive': configdrive_part, + 'root': root_part} + self.mock_ibd.side_effect = iter([True, True, False]) + calls = [mock.call(root_part), + mock.call(swap_part), + mock.call(configdrive_part)] + self.assertRaises(exception.InstanceDeployFailure, + partition_utils.work_on_disk, self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.ephemeral_format, self.image_path, + self.node_uuid, preserve_ephemeral=False, + configdrive=configdrive_url, + boot_option="netboot") + self.assertEqual(self.mock_ibd.call_args_list, calls) + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + configdrive_mb, self.node_uuid, + commit=True, + boot_option="netboot", + boot_mode="bios", + disk_label=None, + cpu_arch="") + mock_unlink.assert_called_once_with('fake-path') + + @mock.patch.object(utils, 'mkfs', lambda fs, path, label=None: None) + @mock.patch.object(disk_utils, 'block_uuid', lambda p: 'uuid') + @mock.patch.object(disk_utils, 'populate_image', autospec=True) + def test_without_image(self, mock_populate): + ephemeral_part = '/dev/fake-part1' + swap_part = '/dev/fake-part2' + root_part = '/dev/fake-part3' + ephemeral_mb = 256 + ephemeral_format = 'exttest' + + self.mock_mp.return_value = {'ephemeral': ephemeral_part, + 'swap': swap_part, + 'root': root_part} + self.mock_ibd.return_value = True + calls = [mock.call(root_part), + mock.call(swap_part), + mock.call(ephemeral_part)] + res = partition_utils.work_on_disk(self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, + ephemeral_format, + None, self.node_uuid) + self.assertEqual(self.mock_ibd.call_args_list, calls) + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, + self.configdrive_mb, + self.node_uuid, commit=True, + boot_option="netboot", + boot_mode="bios", + disk_label=None, + cpu_arch="") + self.assertEqual(root_part, res['partitions']['root']) + self.assertEqual('uuid', res['root uuid']) + self.assertFalse(mock_populate.called) + + @mock.patch.object(utils, 'mkfs', lambda fs, path, label=None: None) + @mock.patch.object(disk_utils, 'block_uuid', lambda p: 'uuid') + @mock.patch.object(disk_utils, 'populate_image', lambda image_path, + root_path, conv_flags=None: None) + def test_gpt_disk_label(self): + ephemeral_part = '/dev/fake-part1' + swap_part = '/dev/fake-part2' + root_part = '/dev/fake-part3' + ephemeral_mb = 256 + ephemeral_format = 'exttest' + + self.mock_mp.return_value = {'ephemeral': ephemeral_part, + 'swap': swap_part, + 'root': root_part} + self.mock_ibd.return_value = True + calls = [mock.call(root_part), + mock.call(swap_part), + mock.call(ephemeral_part)] + partition_utils.work_on_disk(self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, + ephemeral_format, + self.image_path, self.node_uuid, + disk_label='gpt', conv_flags=None) + self.assertEqual(self.mock_ibd.call_args_list, calls) + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, + self.configdrive_mb, + self.node_uuid, commit=True, + boot_option="netboot", + boot_mode="bios", + disk_label='gpt', + cpu_arch="") + + @mock.patch.object(disk_utils, 'block_uuid', autospec=True) + @mock.patch.object(disk_utils, 'populate_image', autospec=True) + @mock.patch.object(utils, 'mkfs', autospec=True) + def test_uefi_localboot(self, mock_mkfs, mock_populate_image, + mock_block_uuid): + """Test that we create a fat filesystem with UEFI localboot.""" + root_part = '/dev/fake-part1' + efi_part = '/dev/fake-part2' + self.mock_mp.return_value = {'root': root_part, + 'efi system partition': efi_part} + self.mock_ibd.return_value = True + mock_ibd_calls = [mock.call(root_part), + mock.call(efi_part)] + + partition_utils.work_on_disk(self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.ephemeral_format, + self.image_path, self.node_uuid, + boot_option="local", boot_mode="uefi") + + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.configdrive_mb, + self.node_uuid, commit=True, + boot_option="local", + boot_mode="uefi", + disk_label=None, + cpu_arch="") + self.assertEqual(self.mock_ibd.call_args_list, mock_ibd_calls) + mock_mkfs.assert_called_once_with(fs='vfat', path=efi_part, + label='efi-part') + mock_populate_image.assert_called_once_with(self.image_path, + root_part, conv_flags=None) + mock_block_uuid.assert_any_call(root_part) + mock_block_uuid.assert_any_call(efi_part) + + @mock.patch.object(disk_utils, 'block_uuid', autospec=True) + @mock.patch.object(disk_utils, 'populate_image', autospec=True) + @mock.patch.object(utils, 'mkfs', autospec=True) + def test_preserve_ephemeral(self, mock_mkfs, mock_populate_image, + mock_block_uuid): + """Test that ephemeral partition doesn't get overwritten.""" + ephemeral_part = '/dev/fake-part1' + root_part = '/dev/fake-part2' + ephemeral_mb = 256 + ephemeral_format = 'exttest' + + self.mock_mp.return_value = {'ephemeral': ephemeral_part, + 'root': root_part} + self.mock_ibd.return_value = True + calls = [mock.call(root_part), + mock.call(ephemeral_part)] + partition_utils.work_on_disk(self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, + ephemeral_format, + self.image_path, self.node_uuid, + preserve_ephemeral=True) + self.assertEqual(self.mock_ibd.call_args_list, calls) + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, + self.configdrive_mb, + self.node_uuid, commit=False, + boot_option="netboot", + boot_mode="bios", + disk_label=None, + cpu_arch="") + self.assertFalse(mock_mkfs.called) + + @mock.patch.object(disk_utils, 'block_uuid', autospec=True) + @mock.patch.object(disk_utils, 'populate_image', autospec=True) + @mock.patch.object(utils, 'mkfs', autospec=True) + def test_ppc64le_prep_part(self, mock_mkfs, mock_populate_image, + mock_block_uuid): + """Test that PReP partition uuid is returned.""" + prep_part = '/dev/fake-part1' + root_part = '/dev/fake-part2' + + self.mock_mp.return_value = {'PReP Boot partition': prep_part, + 'root': root_part} + self.mock_ibd.return_value = True + calls = [mock.call(root_part), + mock.call(prep_part)] + partition_utils.work_on_disk(self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.ephemeral_format, self.image_path, + self.node_uuid, boot_option="local", + cpu_arch='ppc64le') + self.assertEqual(self.mock_ibd.call_args_list, calls) + self.mock_mp.assert_called_once_with(self.dev, self.root_mb, + self.swap_mb, self.ephemeral_mb, + self.configdrive_mb, + self.node_uuid, commit=True, + boot_option="local", + boot_mode="bios", + disk_label=None, + cpu_arch="ppc64le") + self.assertFalse(mock_mkfs.called) + + @mock.patch.object(disk_utils, 'block_uuid', autospec=True) + @mock.patch.object(disk_utils, 'populate_image', autospec=True) + @mock.patch.object(utils, 'mkfs', autospec=True) + def test_convert_to_sparse(self, mock_mkfs, mock_populate_image, + mock_block_uuid): + ephemeral_part = '/dev/fake-part1' + swap_part = '/dev/fake-part2' + root_part = '/dev/fake-part3' + ephemeral_mb = 256 + ephemeral_format = 'exttest' + + self.mock_mp.return_value = {'ephemeral': ephemeral_part, + 'swap': swap_part, + 'root': root_part} + self.mock_ibd.return_value = True + partition_utils.work_on_disk(self.dev, self.root_mb, + self.swap_mb, ephemeral_mb, + ephemeral_format, + self.image_path, self.node_uuid, + disk_label='gpt', conv_flags='sparse') + + mock_populate_image.assert_called_once_with(self.image_path, + root_part, + conv_flags='sparse') + + +class CreateConfigDriveTestCases(base.IronicAgentTest): + + def setUp(self): + super(CreateConfigDriveTestCases, self).setUp() + self.dev = "/dev/fake" + self.config_part_label = "config-2" + self.node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + + @mock.patch.object(utils, 'execute', autospec=True) + @mock.patch.object(utils, 'unlink_without_raise', + autospec=True) + @mock.patch.object(disk_utils, 'dd', + autospec=True) + @mock.patch.object(disk_utils, 'fix_gpt_partition', + autospec=True) + @mock.patch.object(disk_utils, 'get_partition_table_type', + autospec=True) + @mock.patch.object(disk_utils, 'list_partitions', + autospec=True) + @mock.patch.object(partition_utils, 'get_labelled_partition', + autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', + autospec=True) + def test_create_partition_exists(self, mock_get_configdrive, + mock_get_labelled_partition, + mock_list_partitions, mock_table_type, + mock_fix_gpt_partition, + mock_dd, mock_unlink, mock_execute): + config_url = 'http://1.2.3.4/cd' + configdrive_part = '/dev/fake-part1' + configdrive_file = '/tmp/xyz' + configdrive_mb = 10 + + mock_get_labelled_partition.return_value = configdrive_part + mock_get_configdrive.return_value = (configdrive_mb, configdrive_file) + partition_utils.create_config_drive_partition(self.node_uuid, self.dev, + config_url) + mock_fix_gpt_partition.assert_called_with(self.dev, self.node_uuid) + mock_get_configdrive.assert_called_with(config_url, self.node_uuid) + mock_get_labelled_partition.assert_called_with(self.dev, + self.config_part_label, + self.node_uuid) + self.assertFalse(mock_list_partitions.called) + self.assertFalse(mock_execute.called) + self.assertFalse(mock_table_type.called) + mock_dd.assert_called_with(configdrive_file, configdrive_part) + mock_unlink.assert_called_with(configdrive_file) + + @mock.patch.object(utils, 'execute', autospec=True) + @mock.patch.object(utils, 'unlink_without_raise', + autospec=True) + @mock.patch.object(disk_utils, 'dd', + autospec=True) + @mock.patch.object(disk_utils, 'fix_gpt_partition', + autospec=True) + @mock.patch.object(disk_utils, 'get_partition_table_type', + autospec=True) + @mock.patch.object(disk_utils, 'list_partitions', + autospec=True) + @mock.patch.object(partition_utils, 'get_labelled_partition', + autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', + autospec=True) + def test_create_partition_gpt(self, mock_get_configdrive, + mock_get_labelled_partition, + mock_list_partitions, mock_table_type, + mock_fix_gpt_partition, + mock_dd, mock_unlink, mock_execute): + config_url = 'http://1.2.3.4/cd' + configdrive_file = '/tmp/xyz' + configdrive_mb = 10 + + initial_partitions = [{'end': 49152, 'number': 1, 'start': 1, + 'flags': 'boot', 'filesystem': 'ext4', + 'size': 49151}, + {'end': 51099, 'number': 3, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 5, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}] + updated_partitions = [{'end': 49152, 'number': 1, 'start': 1, + 'flags': 'boot', 'filesystem': 'ext4', + 'size': 49151}, + {'end': 51099, 'number': 3, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 4, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 5, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}] + + mock_get_configdrive.return_value = (configdrive_mb, configdrive_file) + mock_get_labelled_partition.return_value = None + + mock_table_type.return_value = 'gpt' + mock_list_partitions.side_effect = [initial_partitions, + updated_partitions] + expected_part = '/dev/fake4' + partition_utils.create_config_drive_partition(self.node_uuid, self.dev, + config_url) + mock_execute.assert_has_calls([ + mock.call('sgdisk', '-n', '0:-64MB:0', self.dev, + run_as_root=True), + mock.call('sync'), + mock.call('udevadm', 'settle'), + mock.call('partprobe', self.dev, attempts=10, run_as_root=True), + mock.call('sgdisk', '-v', self.dev, run_as_root=True), + + mock.call('udevadm', 'settle'), + mock.call('test', '-e', expected_part, attempts=15, + delay_on_retry=True) + ]) + + self.assertEqual(2, mock_list_partitions.call_count) + mock_table_type.assert_called_with(self.dev) + mock_fix_gpt_partition.assert_called_with(self.dev, self.node_uuid) + mock_dd.assert_called_with(configdrive_file, expected_part) + mock_unlink.assert_called_with(configdrive_file) + + @mock.patch.object(disk_utils, 'count_mbr_partitions', autospec=True) + @mock.patch.object(utils, 'execute', autospec=True) + @mock.patch.object(partition_utils.LOG, 'warning', autospec=True) + @mock.patch.object(utils, 'unlink_without_raise', + autospec=True) + @mock.patch.object(disk_utils, 'dd', + autospec=True) + @mock.patch.object(partition_utils, '_is_disk_larger_than_max_size', + autospec=True) + @mock.patch.object(disk_utils, 'fix_gpt_partition', + autospec=True) + @mock.patch.object(disk_utils, 'get_partition_table_type', + autospec=True) + @mock.patch.object(disk_utils, 'list_partitions', + autospec=True) + @mock.patch.object(partition_utils, 'get_labelled_partition', + autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', + autospec=True) + def _test_create_partition_mbr(self, mock_get_configdrive, + mock_get_labelled_partition, + mock_list_partitions, + mock_table_type, + mock_fix_gpt_partition, + mock_disk_exceeds, mock_dd, + mock_unlink, mock_log, mock_execute, + mock_count, disk_size_exceeds_max=False, + is_iscsi_device=False, + is_nvme_device=False): + config_url = 'http://1.2.3.4/cd' + configdrive_file = '/tmp/xyz' + configdrive_mb = 10 + mock_disk_exceeds.return_value = disk_size_exceeds_max + + initial_partitions = [{'end': 49152, 'number': 1, 'start': 1, + 'flags': 'boot', 'filesystem': 'ext4', + 'size': 49151}, + {'end': 51099, 'number': 3, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 5, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}] + updated_partitions = [{'end': 49152, 'number': 1, 'start': 1, + 'flags': 'boot', 'filesystem': 'ext4', + 'size': 49151}, + {'end': 51099, 'number': 3, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 4, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 5, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}] + mock_list_partitions.side_effect = [initial_partitions, + updated_partitions] + # 2 primary partitions, 0 logical partitions + mock_count.return_value = (2, 0) + mock_get_configdrive.return_value = (configdrive_mb, configdrive_file) + mock_get_labelled_partition.return_value = None + + self.node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + if is_iscsi_device: + self.dev = ('/dev/iqn.2008-10.org.openstack:%s.fake' % + self.node_uuid) + expected_part = '%s-part4' % self.dev + elif is_nvme_device: + self.dev = '/dev/nvmefake0' + expected_part = '%sp4' % self.dev + else: + expected_part = '/dev/fake4' + + partition_utils.create_config_drive_partition(self.node_uuid, self.dev, + config_url) + mock_get_configdrive.assert_called_with(config_url, self.node_uuid) + if disk_size_exceeds_max: + self.assertEqual(1, mock_log.call_count) + parted_call = mock.call('parted', '-a', 'optimal', '-s', + '--', self.dev, 'mkpart', + 'primary', 'fat32', 2097087, + 2097151, run_as_root=True) + else: + self.assertEqual(0, mock_log.call_count) + parted_call = mock.call('parted', '-a', 'optimal', '-s', + '--', self.dev, 'mkpart', + 'primary', 'fat32', '-64MiB', + '-0', run_as_root=True) + mock_execute.assert_has_calls([ + parted_call, + mock.call('sync'), + mock.call('udevadm', 'settle'), + mock.call('partprobe', self.dev, attempts=10, run_as_root=True), + mock.call('sgdisk', '-v', self.dev, run_as_root=True), + mock.call('udevadm', 'settle'), + mock.call('test', '-e', expected_part, attempts=15, + delay_on_retry=True) + ]) + self.assertEqual(2, mock_list_partitions.call_count) + mock_table_type.assert_called_with(self.dev) + mock_fix_gpt_partition.assert_called_with(self.dev, self.node_uuid) + mock_disk_exceeds.assert_called_with(self.dev, self.node_uuid) + mock_dd.assert_called_with(configdrive_file, expected_part) + mock_unlink.assert_called_with(configdrive_file) + mock_count.assert_called_with(self.dev) + + def test__create_partition_mbr_disk_under_2TB(self): + self._test_create_partition_mbr(disk_size_exceeds_max=False, + is_iscsi_device=True, + is_nvme_device=False) + + def test__create_partition_mbr_disk_under_2TB_nvme(self): + self._test_create_partition_mbr(disk_size_exceeds_max=False, + is_iscsi_device=False, + is_nvme_device=True) + + def test__create_partition_mbr_disk_exceeds_2TB(self): + self._test_create_partition_mbr(disk_size_exceeds_max=True, + is_iscsi_device=False, + is_nvme_device=False) + + def test__create_partition_mbr_disk_exceeds_2TB_nvme(self): + self._test_create_partition_mbr(disk_size_exceeds_max=True, + is_iscsi_device=False, + is_nvme_device=True) + + @mock.patch.object(disk_utils, 'count_mbr_partitions', autospec=True) + @mock.patch.object(utils, 'execute', autospec=True) + @mock.patch.object(utils, 'unlink_without_raise', + autospec=True) + @mock.patch.object(disk_utils, 'dd', + autospec=True) + @mock.patch.object(partition_utils, '_is_disk_larger_than_max_size', + autospec=True) + @mock.patch.object(disk_utils, 'fix_gpt_partition', + autospec=True) + @mock.patch.object(disk_utils, 'get_partition_table_type', + autospec=True) + @mock.patch.object(disk_utils, 'list_partitions', + autospec=True) + @mock.patch.object(partition_utils, 'get_labelled_partition', + autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', + autospec=True) + def test_create_partition_part_create_fail(self, mock_get_configdrive, + mock_get_labelled_partition, + mock_list_partitions, + mock_table_type, + mock_fix_gpt_partition, + mock_disk_exceeds, mock_dd, + mock_unlink, mock_execute, + mock_count): + config_url = 'http://1.2.3.4/cd' + configdrive_file = '/tmp/xyz' + configdrive_mb = 10 + + initial_partitions = [{'end': 49152, 'number': 1, 'start': 1, + 'flags': 'boot', 'filesystem': 'ext4', + 'size': 49151}, + {'end': 51099, 'number': 3, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 5, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}] + updated_partitions = [{'end': 49152, 'number': 1, 'start': 1, + 'flags': 'boot', 'filesystem': 'ext4', + 'size': 49151}, + {'end': 51099, 'number': 3, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 5, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}] + mock_get_configdrive.return_value = (configdrive_mb, configdrive_file) + mock_get_labelled_partition.return_value = None + mock_disk_exceeds.return_value = False + mock_list_partitions.side_effect = [initial_partitions, + initial_partitions, + updated_partitions] + # 2 primary partitions, 0 logical partitions + mock_count.return_value = (2, 0) + + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'Disk partitioning failed on device', + partition_utils.create_config_drive_partition, + self.node_uuid, self.dev, config_url) + + mock_get_configdrive.assert_called_with(config_url, self.node_uuid) + mock_execute.assert_has_calls([ + mock.call('parted', '-a', 'optimal', '-s', '--', + self.dev, 'mkpart', 'primary', + 'fat32', '-64MiB', '-0', + run_as_root=True), + mock.call('sync'), + mock.call('udevadm', 'settle'), + mock.call('partprobe', self.dev, attempts=10, run_as_root=True), + mock.call('sgdisk', '-v', self.dev, run_as_root=True), + ]) + + self.assertEqual(2, mock_list_partitions.call_count) + mock_fix_gpt_partition.assert_called_with(self.dev, self.node_uuid) + mock_table_type.assert_called_with(self.dev) + mock_disk_exceeds.assert_called_with(self.dev, self.node_uuid) + self.assertFalse(mock_dd.called) + mock_unlink.assert_called_with(configdrive_file) + mock_count.assert_called_once_with(self.dev) + + @mock.patch.object(disk_utils, 'count_mbr_partitions', autospec=True) + @mock.patch.object(utils, 'execute', autospec=True) + @mock.patch.object(utils, 'unlink_without_raise', + autospec=True) + @mock.patch.object(disk_utils, 'dd', + autospec=True) + @mock.patch.object(partition_utils, '_is_disk_larger_than_max_size', + autospec=True) + @mock.patch.object(disk_utils, 'fix_gpt_partition', + autospec=True) + @mock.patch.object(disk_utils, 'get_partition_table_type', + autospec=True) + @mock.patch.object(disk_utils, 'list_partitions', + autospec=True) + @mock.patch.object(partition_utils, 'get_labelled_partition', + autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', + autospec=True) + def test_create_partition_part_create_exc(self, mock_get_configdrive, + mock_get_labelled_partition, + mock_list_partitions, + mock_table_type, + mock_fix_gpt_partition, + mock_disk_exceeds, mock_dd, + mock_unlink, mock_execute, + mock_count): + config_url = 'http://1.2.3.4/cd' + configdrive_file = '/tmp/xyz' + configdrive_mb = 10 + + initial_partitions = [{'end': 49152, 'number': 1, 'start': 1, + 'flags': 'boot', 'filesystem': 'ext4', + 'size': 49151}, + {'end': 51099, 'number': 3, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 5, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}] + mock_get_configdrive.return_value = (configdrive_mb, configdrive_file) + mock_get_labelled_partition.return_value = None + mock_disk_exceeds.return_value = False + mock_list_partitions.side_effect = [initial_partitions, + initial_partitions] + # 2 primary partitions, 0 logical partitions + mock_count.return_value = (2, 0) + + mock_execute.side_effect = processutils.ProcessExecutionError + + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'Failed to create config drive on disk', + partition_utils.create_config_drive_partition, + self.node_uuid, self.dev, config_url) + + mock_get_configdrive.assert_called_with(config_url, self.node_uuid) + mock_execute.assert_called_with('parted', '-a', 'optimal', '-s', '--', + self.dev, 'mkpart', 'primary', + 'fat32', '-64MiB', '-0', + run_as_root=True) + self.assertEqual(1, mock_list_partitions.call_count) + mock_fix_gpt_partition.assert_called_with(self.dev, self.node_uuid) + mock_table_type.assert_called_with(self.dev) + mock_disk_exceeds.assert_called_with(self.dev, self.node_uuid) + self.assertFalse(mock_dd.called) + mock_unlink.assert_called_with(configdrive_file) + mock_count.assert_called_once_with(self.dev) + + @mock.patch.object(disk_utils, 'count_mbr_partitions', autospec=True) + @mock.patch.object(utils, 'unlink_without_raise', + autospec=True) + @mock.patch.object(disk_utils, 'dd', + autospec=True) + @mock.patch.object(disk_utils, 'fix_gpt_partition', + autospec=True) + @mock.patch.object(disk_utils, 'get_partition_table_type', + autospec=True) + @mock.patch.object(disk_utils, 'list_partitions', + autospec=True) + @mock.patch.object(partition_utils, 'get_labelled_partition', + autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', + autospec=True) + def test_create_partition_num_parts_exceed(self, mock_get_configdrive, + mock_get_labelled_partition, + mock_list_partitions, + mock_table_type, + mock_fix_gpt_partition, + mock_dd, mock_unlink, + mock_count): + config_url = 'http://1.2.3.4/cd' + configdrive_file = '/tmp/xyz' + configdrive_mb = 10 + + partitions = [{'end': 49152, 'number': 1, 'start': 1, + 'flags': 'boot', 'filesystem': 'ext4', + 'size': 49151}, + {'end': 51099, 'number': 2, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 3, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}, + {'end': 51099, 'number': 4, 'start': 49153, + 'flags': '', 'filesystem': '', 'size': 2046}] + mock_get_configdrive.return_value = (configdrive_mb, configdrive_file) + mock_get_labelled_partition.return_value = None + mock_list_partitions.side_effect = [partitions, partitions] + # 4 primary partitions, 0 logical partitions + mock_count.return_value = (4, 0) + + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'Config drive cannot be created for node', + partition_utils.create_config_drive_partition, + self.node_uuid, self.dev, config_url) + + mock_get_configdrive.assert_called_with(config_url, self.node_uuid) + self.assertEqual(1, mock_list_partitions.call_count) + mock_fix_gpt_partition.assert_called_with(self.dev, self.node_uuid) + mock_table_type.assert_called_with(self.dev) + self.assertFalse(mock_dd.called) + mock_unlink.assert_called_with(configdrive_file) + mock_count.assert_called_once_with(self.dev) + + @mock.patch.object(utils, 'execute', autospec=True) + @mock.patch.object(utils, 'unlink_without_raise', + autospec=True) + @mock.patch.object(partition_utils, 'get_labelled_partition', + autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', + autospec=True) + def test_create_partition_conf_drive_sz_exceed(self, mock_get_configdrive, + mock_get_labelled_partition, + mock_unlink, mock_execute): + config_url = 'http://1.2.3.4/cd' + configdrive_file = '/tmp/xyz' + configdrive_mb = 65 + + mock_get_configdrive.return_value = (configdrive_mb, configdrive_file) + mock_get_labelled_partition.return_value = None + + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'Config drive size exceeds maximum limit', + partition_utils.create_config_drive_partition, + self.node_uuid, self.dev, config_url) + + mock_get_configdrive.assert_called_with(config_url, self.node_uuid) + mock_unlink.assert_called_with(configdrive_file) + + @mock.patch.object(disk_utils, 'count_mbr_partitions', autospec=True) + @mock.patch.object(utils, 'execute', autospec=True) + @mock.patch.object(utils, 'unlink_without_raise', + autospec=True) + @mock.patch.object(disk_utils, 'fix_gpt_partition', + autospec=True) + @mock.patch.object(disk_utils, 'get_partition_table_type', + autospec=True) + @mock.patch.object(partition_utils, 'get_labelled_partition', + autospec=True) + @mock.patch.object(partition_utils, 'get_configdrive', + autospec=True) + def test_create_partition_conf_drive_error_counting( + self, mock_get_configdrive, mock_get_labelled_partition, + mock_table_type, mock_fix_gpt_partition, + mock_unlink, mock_execute, mock_count): + config_url = 'http://1.2.3.4/cd' + configdrive_file = '/tmp/xyz' + configdrive_mb = 10 + + mock_get_configdrive.return_value = (configdrive_mb, configdrive_file) + mock_get_labelled_partition.return_value = None + mock_count.side_effect = ValueError('Booooom') + + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'Failed to check the number of primary ', + partition_utils.create_config_drive_partition, + self.node_uuid, self.dev, config_url) + + mock_get_configdrive.assert_called_with(config_url, self.node_uuid) + mock_unlink.assert_called_with(configdrive_file) + mock_fix_gpt_partition.assert_called_with(self.dev, self.node_uuid) + mock_table_type.assert_called_with(self.dev) + mock_count.assert_called_once_with(self.dev)