From f824930bbdd2521607e6eb50cf7bb1f2160b40d9 Mon Sep 17 00:00:00 2001 From: Dmitry Tantsur Date: Mon, 11 Mar 2024 17:29:58 +0100 Subject: [PATCH] Import disk_{utils,partitioner} from ironic-lib With the iscsi deploy long gone, these modules are only used in IPA and in fact represent a large part of its critical logic. Having them separately sometimes makes fixing issues tricky if an interface of a function needs changing. This change imports the code mostly as it is, just removing run_as_root and a deprecated function, as well as moving configuration options to config.py. Also migrates one relevant function from ironic_lib.utils. Change-Id: If8fae8210d85c61abb85c388b300e40a75d0531c --- ironic_python_agent/config.py | 46 +- ironic_python_agent/disk_partitioner.py | 124 +++ ironic_python_agent/disk_utils.py | 709 ++++++++++++++ ironic_python_agent/efi_utils.py | 2 +- ironic_python_agent/extensions/standby.py | 21 +- ironic_python_agent/hardware.py | 19 +- ironic_python_agent/inject_files.py | 2 +- ironic_python_agent/partition_utils.py | 2 +- ironic_python_agent/raid_utils.py | 2 +- .../tests/unit/extensions/test_image.py | 2 +- .../tests/unit/extensions/test_standby.py | 84 +- .../tests/unit/test_disk_partitioner.py | 202 ++++ .../tests/unit/test_disk_utils.py | 927 ++++++++++++++++++ .../tests/unit/test_efi_utils.py | 2 +- .../tests/unit/test_hardware.py | 18 +- .../tests/unit/test_inject_files.py | 2 +- .../tests/unit/test_partition_utils.py | 37 +- .../tests/unit/test_raid_utils.py | 2 +- 18 files changed, 2095 insertions(+), 108 deletions(-) create mode 100644 ironic_python_agent/disk_partitioner.py create mode 100644 ironic_python_agent/disk_utils.py create mode 100644 ironic_python_agent/tests/unit/test_disk_partitioner.py create mode 100644 ironic_python_agent/tests/unit/test_disk_utils.py diff --git a/ironic_python_agent/config.py b/ironic_python_agent/config.py index 6de4da826..f311bf187 100644 --- a/ironic_python_agent/config.py +++ b/ironic_python_agent/config.py @@ -372,11 +372,55 @@ cli_opts = [ 'determine if this action is necessary.'), ] +disk_utils_opts = [ + cfg.IntOpt('efi_system_partition_size', + default=550, + help='Size of EFI system partition in MiB when configuring ' + 'UEFI systems for local boot. A common minimum is ~200 ' + 'megabytes, however OS driven firmware updates and ' + 'unikernel usage generally requires more space on the ' + 'efi partition.'), + cfg.IntOpt('bios_boot_partition_size', + default=1, + help='Size of BIOS Boot partition in MiB when configuring ' + 'GPT partitioned systems for local boot in BIOS.'), + cfg.StrOpt('dd_block_size', + default='1M', + help='Block size to use when writing to the nodes disk.'), + cfg.IntOpt('partition_detection_attempts', + default=3, + min=1, + help='Maximum attempts to detect a newly created partition.'), + cfg.IntOpt('partprobe_attempts', + default=10, + help='Maximum number of attempts to try to read the ' + 'partition.'), +] + +disk_part_opts = [ + cfg.IntOpt('check_device_interval', + default=1, + help='After Ironic has completed creating the partition table, ' + 'it continues to check for activity on the attached iSCSI ' + 'device status at this interval prior to copying the image' + ' to the node, in seconds'), + cfg.IntOpt('check_device_max_retries', + default=20, + help='The maximum number of times to check that the device is ' + 'not accessed by another process. If the device is still ' + 'busy after that, the disk partitioning will be treated as' + ' having failed.') +] + CONF.register_cli_opts(cli_opts) +CONF.register_opts(disk_utils_opts, group='disk_utils') +CONF.register_opts(disk_part_opts, group='disk_partitioner') def list_opts(): - return [('DEFAULT', cli_opts)] + return [('DEFAULT', cli_opts), + ('disk_utils', disk_utils_opts), + ('disk_partitioner', disk_part_opts)] def override(params): diff --git a/ironic_python_agent/disk_partitioner.py b/ironic_python_agent/disk_partitioner.py new file mode 100644 index 000000000..a37a8b67b --- /dev/null +++ b/ironic_python_agent/disk_partitioner.py @@ -0,0 +1,124 @@ +# Copyright 2014 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Code for creating partitions on a disk. + +Imported from ironic-lib's disk_utils as of the following commit: +https://opendev.org/openstack/ironic-lib/commit/42fa5d63861ba0f04b9a4f67212173d7013a1332 +""" + +import logging + +from ironic_lib.common.i18n import _ +from ironic_lib import exception +from ironic_lib import utils +from oslo_config import cfg + +CONF = cfg.CONF + +LOG = logging.getLogger(__name__) + + +class DiskPartitioner(object): + + def __init__(self, device, disk_label='msdos', alignment='optimal'): + """A convenient wrapper around the parted tool. + + :param device: The device path. + :param disk_label: The type of the partition table. Valid types are: + "bsd", "dvh", "gpt", "loop", "mac", "msdos", + "pc98", or "sun". + :param alignment: Set alignment for newly created partitions. + Valid types are: none, cylinder, minimal and + optimal. + + """ + self._device = device + self._disk_label = disk_label + self._alignment = alignment + self._partitions = [] + + def _exec(self, *args): + # NOTE(lucasagomes): utils.execute() is already a wrapper on top + # of processutils.execute() which raises specific + # exceptions. It also logs any failure so we don't + # need to log it again here. + utils.execute('parted', '-a', self._alignment, '-s', self._device, + '--', 'unit', 'MiB', *args, use_standard_locale=True) + + def add_partition(self, size, part_type='primary', fs_type='', + boot_flag=None, extra_flags=None): + """Add a partition. + + :param size: The size of the partition in MiB. + :param part_type: The type of the partition. Valid values are: + primary, logical, or extended. + :param fs_type: The filesystem type. Valid types are: ext2, fat32, + fat16, HFS, linux-swap, NTFS, reiserfs, ufs. + If blank (''), it will create a Linux native + partition (83). + :param boot_flag: Boot flag that needs to be configured on the + partition. Ignored if None. It can take values + 'bios_grub', 'boot'. + :param extra_flags: List of flags to set on the partition. Ignored + if None. + :returns: The partition number. + + """ + self._partitions.append({'size': size, + 'type': part_type, + 'fs_type': fs_type, + 'boot_flag': boot_flag, + 'extra_flags': extra_flags}) + return len(self._partitions) + + def get_partitions(self): + """Get the partitioning layout. + + :returns: An iterator with the partition number and the + partition layout. + + """ + return enumerate(self._partitions, 1) + + def commit(self): + """Write to the disk.""" + LOG.debug("Committing partitions to disk.") + cmd_args = ['mklabel', self._disk_label] + # NOTE(lucasagomes): Lead in with 1MiB to allow room for the + # partition table itself. + start = 1 + for num, part in self.get_partitions(): + end = start + part['size'] + cmd_args.extend(['mkpart', part['type'], part['fs_type'], + str(start), str(end)]) + if part['boot_flag']: + cmd_args.extend(['set', str(num), part['boot_flag'], 'on']) + if part['extra_flags']: + for flag in part['extra_flags']: + cmd_args.extend(['set', str(num), flag, 'on']) + start = end + + self._exec(*cmd_args) + + try: + from ironic_python_agent import disk_utils # circular dependency + disk_utils.wait_for_disk_to_become_available(self._device) + except exception.IronicException as e: + raise exception.InstanceDeployFailure( + _('Disk partitioning failed on device %(device)s. ' + 'Error: %(error)s') + % {'device': self._device, 'error': e}) diff --git a/ironic_python_agent/disk_utils.py b/ironic_python_agent/disk_utils.py new file mode 100644 index 000000000..dfabc3fc4 --- /dev/null +++ b/ironic_python_agent/disk_utils.py @@ -0,0 +1,709 @@ +# Copyright 2014 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Various utilities related to disk handling. + +Imported from ironic-lib's disk_utils as of the following commit: +https://opendev.org/openstack/ironic-lib/commit/42fa5d63861ba0f04b9a4f67212173d7013a1332 +""" + +import logging +import os +import re +import stat +import time + +from ironic_lib.common.i18n import _ +from ironic_lib import exception +from ironic_lib import qemu_img +from ironic_lib import utils +from oslo_concurrency import processutils +from oslo_config import cfg +from oslo_utils import excutils +import tenacity + +from ironic_python_agent import disk_partitioner + +CONF = cfg.CONF + +LOG = logging.getLogger(__name__) + +_PARTED_PRINT_RE = re.compile(r"^(\d+):([\d\.]+)MiB:" + r"([\d\.]+)MiB:([\d\.]+)MiB:(\w*):(.*):(.*);") +_PARTED_TABLE_TYPE_RE = re.compile(r'^.*partition\s+table\s*:\s*(gpt|msdos)', + re.IGNORECASE | re.MULTILINE) + +CONFIGDRIVE_LABEL = "config-2" +MAX_CONFIG_DRIVE_SIZE_MB = 64 + +GPT_SIZE_SECTORS = 33 + +# Maximum disk size supported by MBR is 2TB (2 * 1024 * 1024 MB) +MAX_DISK_SIZE_MB_SUPPORTED_BY_MBR = 2097152 + + +def list_partitions(device): + """Get partitions information from given device. + + :param device: The device path. + :returns: list of dictionaries (one per partition) with keys: + number, start, end, size (in MiB), filesystem, partition_name, + flags, path. + """ + output = utils.execute( + 'parted', '-s', '-m', device, 'unit', 'MiB', 'print', + use_standard_locale=True)[0] + if isinstance(output, bytes): + output = output.decode("utf-8") + lines = [line for line in output.split('\n') if line.strip()][2:] + # Example of line: 1:1.00MiB:501MiB:500MiB:ext4::boot + fields = ('number', 'start', 'end', 'size', 'filesystem', 'partition_name', + 'flags') + result = [] + for line in lines: + match = _PARTED_PRINT_RE.match(line) + if match is None: + LOG.warning("Partition information from parted for device " + "%(device)s does not match " + "expected format: %(line)s", + dict(device=device, line=line)) + continue + # Cast int fields to ints (some are floats and we round them down) + groups = [int(float(x)) if i < 4 else x + for i, x in enumerate(match.groups())] + item = dict(zip(fields, groups)) + item['path'] = partition_index_to_path(device, item['number']) + result.append(item) + return result + + +def count_mbr_partitions(device): + """Count the number of primary and logical partitions on a MBR + + :param device: The device path. + :returns: A tuple with the number of primary partitions and logical + partitions. + :raise: ValueError if the device does not have a valid MBR partition + table. + """ + # -d do not update the kernel table + # -s print a summary of the partition table + output, err = utils.execute('partprobe', '-d', '-s', device, + use_standard_locale=True) + if 'msdos' not in output: + raise ValueError('The device %s does not have a valid MBR ' + 'partition table' % device) + # Sample output: /dev/vdb: msdos partitions 1 2 3 <5 6 7> + # The partitions with number > 4 (and inside <>) are logical partitions + output = output.replace('<', '').replace('>', '') + partitions = [int(s) for s in output.split() if s.isdigit()] + + return (sum(i < 5 for i in partitions), sum(i > 4 for i in partitions)) + + +def get_disk_identifier(dev): + """Get the disk identifier from the disk being exposed by the ramdisk. + + This disk identifier is appended to the pxe config which will then be + used by chain.c32 to detect the correct disk to chainload. This is helpful + in deployments to nodes with multiple disks. + + http://www.syslinux.org/wiki/index.php/Comboot/chain.c32#mbr: + + :param dev: Path for the already populated disk device. + :raises OSError: When the hexdump binary is unavailable. + :returns: The Disk Identifier. + """ + disk_identifier = utils.execute('hexdump', '-s', '440', '-n', '4', + '-e', '''\"0x%08x\"''', + dev, attempts=5, delay_on_retry=True) + return disk_identifier[0] + + +def get_partition_table_type(device): + """Get partition table type, msdos or gpt. + + :param device: the name of the device + :return: dos, gpt or None + """ + out = utils.execute('parted', '--script', device, '--', 'print', + use_standard_locale=True)[0] + m = _PARTED_TABLE_TYPE_RE.search(out) + if m: + return m.group(1) + + LOG.warning("Unable to get partition table type for device %s", device) + return 'unknown' + + +def _blkid(device, probe=False, fields=None): + args = [] + if probe: + args.append('-p') + if fields: + args += sum((['-s', field] for field in fields), []) + + output, err = utils.execute('blkid', device, *args, + use_standard_locale=True) + if output.strip(): + return output.split(': ', 1)[1] + else: + return "" + + +def _lsblk(device, deps=True, fields=None): + args = ['--pairs', '--bytes', '--ascii'] + if not deps: + args.append('--nodeps') + if fields: + args.extend(['--output', ','.join(fields)]) + else: + args.append('--output-all') + + output, err = utils.execute('lsblk', device, *args, + use_standard_locale=True) + return output.strip() + + +def get_device_information(device, fields=None): + """Get information about a device using blkid. + + Can be applied to all block devices: disks, RAID, partitions. + + :param device: Device name. + :param fields: A list of fields to request (all by default). + :return: A dictionary with requested fields as keys. + :raises: ProcessExecutionError + """ + output = _lsblk(device, fields=fields, deps=False) + if output: + return next(utils.parse_device_tags(output)) + else: + return {} + + +def find_efi_partition(device): + """Looks for the EFI partition on a given device. + + A boot partition on a GPT disk is assumed to be an EFI partition as well. + + :param device: the name of the device + :return: the EFI partition record from `list_partitions` or None + """ + is_gpt = get_partition_table_type(device) == 'gpt' + for part in list_partitions(device): + flags = {x.strip() for x in part['flags'].split(',')} + if 'esp' in flags or ('boot' in flags and is_gpt): + LOG.debug("Found EFI partition %s on device %s", part, device) + return part + else: + LOG.debug("No efi partition found on device %s", device) + + +_ISCSI_PREFIX = "iqn.2008-10.org.openstack:" + + +def is_last_char_digit(dev): + """check whether device name ends with a digit""" + if len(dev) >= 1: + return dev[-1].isdigit() + return False + + +def partition_index_to_path(device, index): + """Guess a partition path based on its device and index. + + :param device: Device path. + :param index: Partition index. + """ + # the actual device names in the baremetal are like /dev/sda, /dev/sdb etc. + # While for the iSCSI device, the naming convention has a format which has + # iqn also embedded in it. + # When this function is called by ironic-conductor, the iSCSI device name + # should be appended by "part%d". While on the baremetal, it should name + # the device partitions as /dev/sda1 and not /dev/sda-part1. + if _ISCSI_PREFIX in device: + part_template = '%s-part%d' + elif is_last_char_digit(device): + part_template = '%sp%d' + else: + part_template = '%s%d' + return part_template % (device, index) + + +def make_partitions(dev, root_mb, swap_mb, ephemeral_mb, + configdrive_mb, node_uuid, commit=True, + boot_option="netboot", boot_mode="bios", + disk_label=None, cpu_arch=""): + """Partition the disk device. + + Create partitions for root, swap, ephemeral and configdrive on a + disk device. + + :param dev: Path for the device to work on. + :param root_mb: Size of the root partition in mebibytes (MiB). + :param swap_mb: Size of the swap partition in mebibytes (MiB). If 0, + no partition will be created. + :param ephemeral_mb: Size of the ephemeral partition in mebibytes (MiB). + If 0, no partition will be created. + :param configdrive_mb: Size of the configdrive partition in + mebibytes (MiB). If 0, no partition will be created. + :param commit: True/False. Default for this setting is True. If False + partitions will not be written to disk. + :param boot_option: Can be "local" or "netboot". "netboot" by default. + :param boot_mode: Can be "bios" or "uefi". "bios" by default. + :param node_uuid: Node's uuid. Used for logging. + :param disk_label: The disk label to be used when creating the + partition table. Valid values are: "msdos", "gpt" or None; If None + Ironic will figure it out according to the boot_mode parameter. + :param cpu_arch: Architecture of the node the disk device belongs to. + When using the default value of None, no architecture specific + steps will be taken. This default should be used for x86_64. When + set to ppc64*, architecture specific steps are taken for booting a + partition image locally. + :returns: A dictionary containing the partition type as Key and partition + path as Value for the partitions created by this method. + + """ + LOG.debug("Starting to partition the disk device: %(dev)s " + "for node %(node)s", + {'dev': dev, 'node': node_uuid}) + part_dict = {} + + if disk_label is None: + disk_label = 'gpt' if boot_mode == 'uefi' else 'msdos' + + dp = disk_partitioner.DiskPartitioner(dev, disk_label=disk_label) + + # For uefi localboot, switch partition table to gpt and create the efi + # system partition as the first partition. + if boot_mode == "uefi" and boot_option == "local": + part_num = dp.add_partition(CONF.disk_utils.efi_system_partition_size, + fs_type='fat32', + boot_flag='boot') + part_dict['efi system partition'] = partition_index_to_path( + dev, part_num) + + if (boot_mode == "bios" and boot_option == "local" and disk_label == "gpt" + and not cpu_arch.startswith('ppc64')): + part_num = dp.add_partition(CONF.disk_utils.bios_boot_partition_size, + boot_flag='bios_grub') + part_dict['BIOS Boot partition'] = partition_index_to_path( + dev, part_num) + + # NOTE(mjturek): With ppc64* nodes, partition images are expected to have + # a PrEP partition at the start of the disk. This is an 8 MiB partition + # with the boot and prep flags set. The bootloader should be installed + # here. + if (cpu_arch.startswith("ppc64") and boot_mode == "bios" + and boot_option == "local"): + LOG.debug("Add PReP boot partition (8 MB) to device: " + "%(dev)s for node %(node)s", + {'dev': dev, 'node': node_uuid}) + boot_flag = 'boot' if disk_label == 'msdos' else None + part_num = dp.add_partition(8, part_type='primary', + boot_flag=boot_flag, extra_flags=['prep']) + part_dict['PReP Boot partition'] = partition_index_to_path( + dev, part_num) + if ephemeral_mb: + LOG.debug("Add ephemeral partition (%(size)d MB) to device: %(dev)s " + "for node %(node)s", + {'dev': dev, 'size': ephemeral_mb, 'node': node_uuid}) + part_num = dp.add_partition(ephemeral_mb) + part_dict['ephemeral'] = partition_index_to_path(dev, part_num) + if swap_mb: + LOG.debug("Add Swap partition (%(size)d MB) to device: %(dev)s " + "for node %(node)s", + {'dev': dev, 'size': swap_mb, 'node': node_uuid}) + part_num = dp.add_partition(swap_mb, fs_type='linux-swap') + part_dict['swap'] = partition_index_to_path(dev, part_num) + if configdrive_mb: + LOG.debug("Add config drive partition (%(size)d MB) to device: " + "%(dev)s for node %(node)s", + {'dev': dev, 'size': configdrive_mb, 'node': node_uuid}) + part_num = dp.add_partition(configdrive_mb) + part_dict['configdrive'] = partition_index_to_path(dev, part_num) + + # NOTE(lucasagomes): Make the root partition the last partition. This + # enables tools like cloud-init's growroot utility to expand the root + # partition until the end of the disk. + LOG.debug("Add root partition (%(size)d MB) to device: %(dev)s " + "for node %(node)s", + {'dev': dev, 'size': root_mb, 'node': node_uuid}) + + boot_val = 'boot' if (not cpu_arch.startswith("ppc64") + and boot_mode == "bios" + and boot_option == "local" + and disk_label == "msdos") else None + + part_num = dp.add_partition(root_mb, boot_flag=boot_val) + + part_dict['root'] = partition_index_to_path(dev, part_num) + + if commit: + # write to the disk + dp.commit() + trigger_device_rescan(dev) + return part_dict + + +def is_block_device(dev): + """Check whether a device is block or not.""" + attempts = CONF.disk_utils.partition_detection_attempts + for attempt in range(attempts): + try: + s = os.stat(dev) + except OSError as e: + LOG.debug("Unable to stat device %(dev)s. Attempt %(attempt)d " + "out of %(total)d. Error: %(err)s", + {"dev": dev, "attempt": attempt + 1, + "total": attempts, "err": e}) + time.sleep(1) + else: + return stat.S_ISBLK(s.st_mode) + msg = _("Unable to stat device %(dev)s after attempting to verify " + "%(attempts)d times.") % {'dev': dev, 'attempts': attempts} + LOG.error(msg) + raise exception.InstanceDeployFailure(msg) + + +def dd(src, dst, conv_flags=None): + """Execute dd from src to dst.""" + if conv_flags: + extra_args = ['conv=%s' % conv_flags] + else: + extra_args = [] + + utils.dd(src, dst, 'bs=%s' % CONF.disk_utils.dd_block_size, 'oflag=direct', + *extra_args) + + +def populate_image(src, dst, conv_flags=None): + data = qemu_img.image_info(src) + if data.file_format == 'raw': + dd(src, dst, conv_flags=conv_flags) + else: + qemu_img.convert_image(src, dst, 'raw', True, sparse_size='0') + + +def block_uuid(dev): + """Get UUID of a block device. + + Try to fetch the UUID, if that fails, try to fetch the PARTUUID. + """ + info = get_device_information(dev, fields=['UUID', 'PARTUUID']) + if info.get('UUID'): + return info['UUID'] + else: + LOG.debug('Falling back to partition UUID as the block device UUID ' + 'was not found while examining %(device)s', + {'device': dev}) + return info.get('PARTUUID', '') + + +def get_image_mb(image_path, virtual_size=True): + """Get size of an image in Megabyte.""" + mb = 1024 * 1024 + if not virtual_size: + image_byte = os.path.getsize(image_path) + else: + data = qemu_img.image_info(image_path) + image_byte = data.virtual_size + + # round up size to MB + image_mb = int((image_byte + mb - 1) / mb) + return image_mb + + +def get_dev_block_size(dev): + """Get the device size in 512 byte sectors.""" + block_sz, cmderr = utils.execute('blockdev', '--getsz', dev) + return int(block_sz) + + +def destroy_disk_metadata(dev, node_uuid): + """Destroy metadata structures on node's disk. + + Ensure that node's disk magic strings are wiped without zeroing the + entire drive. To do this we use the wipefs tool from util-linux. + + :param dev: Path for the device to work on. + :param node_uuid: Node's uuid. Used for logging. + """ + # NOTE(NobodyCam): This is needed to work around bug: + # https://bugs.launchpad.net/ironic/+bug/1317647 + LOG.debug("Start destroy disk metadata for node %(node)s.", + {'node': node_uuid}) + try: + utils.execute('wipefs', '--force', '--all', dev, + use_standard_locale=True) + except processutils.ProcessExecutionError as e: + with excutils.save_and_reraise_exception() as ctxt: + # NOTE(zhenguo): Check if --force option is supported for wipefs, + # if not, we should try without it. + if '--force' in str(e): + ctxt.reraise = False + utils.execute('wipefs', '--all', dev, + use_standard_locale=True) + # NOTE(TheJulia): sgdisk attempts to load and make sense of the + # partition tables in advance of wiping the partition data. + # This means when a CRC error is found, sgdisk fails before + # erasing partition data. + # This is the same bug as + # https://bugs.launchpad.net/ironic-python-agent/+bug/1737556 + + # Overwrite the Primary GPT, catch very small partitions (like EBRs) + dd_device = 'of=%s' % dev + dd_count = 'count=%s' % GPT_SIZE_SECTORS + dev_size = get_dev_block_size(dev) + if dev_size < GPT_SIZE_SECTORS: + dd_count = 'count=%s' % dev_size + utils.execute('dd', 'bs=512', 'if=/dev/zero', dd_device, dd_count, + 'oflag=direct', use_standard_locale=True) + + # Overwrite the Secondary GPT, do this only if there could be one + if dev_size > GPT_SIZE_SECTORS: + gpt_backup = dev_size - GPT_SIZE_SECTORS + dd_seek = 'seek=%i' % gpt_backup + dd_count = 'count=%s' % GPT_SIZE_SECTORS + utils.execute('dd', 'bs=512', 'if=/dev/zero', dd_device, dd_count, + 'oflag=direct', dd_seek, use_standard_locale=True) + + # Go ahead and let sgdisk run as well. + utils.execute('sgdisk', '-Z', dev, use_standard_locale=True) + + try: + wait_for_disk_to_become_available(dev) + except exception.IronicException as e: + raise exception.InstanceDeployFailure( + _('Destroying metadata failed on device %(device)s. ' + 'Error: %(error)s') + % {'device': dev, 'error': e}) + + LOG.info("Disk metadata on %(dev)s successfully destroyed for node " + "%(node)s", {'dev': dev, 'node': node_uuid}) + + +def _fix_gpt_structs(device, node_uuid): + """Checks backup GPT data structures and moves them to end of the device + + :param device: The device path. + :param node_uuid: UUID of the Node. Used for logging. + :raises: InstanceDeployFailure, if any disk partitioning related + commands fail. + """ + try: + output, _err = utils.execute('sgdisk', '-v', device) + + search_str = "it doesn't reside\nat the end of the disk" + if search_str in output: + utils.execute('sgdisk', '-e', device) + except (processutils.UnknownArgumentError, + processutils.ProcessExecutionError, OSError) as e: + msg = (_('Failed to fix GPT data structures on disk %(disk)s ' + 'for node %(node)s. Error: %(error)s') % + {'disk': device, 'node': node_uuid, 'error': e}) + LOG.error(msg) + raise exception.InstanceDeployFailure(msg) + + +def fix_gpt_partition(device, node_uuid): + """Fix GPT partition + + Fix GPT table information when image is written to a disk which + has a bigger extend (e.g. 30GB image written on a 60Gb physical disk). + + :param device: The device path. + :param node_uuid: UUID of the Node. + :raises: InstanceDeployFailure if exception is caught. + """ + try: + disk_is_gpt_partitioned = (get_partition_table_type(device) == 'gpt') + if disk_is_gpt_partitioned: + _fix_gpt_structs(device, node_uuid) + except Exception as e: + msg = (_('Failed to fix GPT partition on disk %(disk)s ' + 'for node %(node)s. Error: %(error)s') % + {'disk': device, 'node': node_uuid, 'error': e}) + LOG.error(msg) + raise exception.InstanceDeployFailure(msg) + + +def udev_settle(): + """Wait for the udev event queue to settle. + + Wait for the udev event queue to settle to make sure all devices + are detected once the machine boots up. + + :return: True on success, False otherwise. + """ + LOG.debug('Waiting until udev event queue is empty') + try: + utils.execute('udevadm', 'settle') + except processutils.ProcessExecutionError as e: + LOG.warning('Something went wrong when waiting for udev ' + 'to settle. Error: %s', e) + return False + else: + return True + + +def partprobe(device, attempts=None): + """Probe partitions on the given device. + + :param device: The block device containing partitions that is attempting + to be updated. + :param attempts: Number of attempts to run partprobe, the default is read + from the configuration. + :return: True on success, False otherwise. + """ + if attempts is None: + attempts = CONF.disk_utils.partprobe_attempts + + try: + utils.execute('partprobe', device, attempts=attempts) + except (processutils.UnknownArgumentError, + processutils.ProcessExecutionError, OSError) as e: + LOG.warning("Unable to probe for partitions on device %(device)s, " + "the partitioning table may be broken. Error: %(error)s", + {'device': device, 'error': e}) + return False + else: + return True + + +def trigger_device_rescan(device, attempts=None): + """Sync and trigger device rescan. + + Disk partition performed via parted, when performed on a ramdisk + do not have to honor the fsync mechanism. In essence, fsync is used + on the file representing the block device, which falls to the kernel + filesystem layer to trigger a sync event. On a ramdisk using ramfs, + this is an explicit non-operation. + + As a result of this, we need to trigger a system wide sync operation + which will trigger cache to flush to disk, after which partition changes + should be visible upon re-scan. + + When ramdisks are not in use, this also helps ensure that data has + been safely flushed across the wire, such as on iscsi connections. + + :param device: The block device containing partitions that is attempting + to be updated. + :param attempts: Number of attempts to run partprobe, the default is read + from the configuration. + :return: True on success, False otherwise. + """ + LOG.debug('Explicitly calling sync to force buffer/cache flush') + utils.execute('sync') + # Make sure any additions to the partitioning are reflected in the + # kernel. + udev_settle() + partprobe(device, attempts=attempts) + udev_settle() + try: + # Also verify that the partitioning is correct now. + utils.execute('sgdisk', '-v', device) + except processutils.ProcessExecutionError as exc: + LOG.warning('Failed to verify partition tables on device %(dev)s: ' + '%(err)s', {'dev': device, 'err': exc}) + return False + else: + return True + + +# NOTE(dtantsur): this function was in ironic_lib.utils before migration +# (presumably to avoid a circular dependency with disk_partitioner) +def wait_for_disk_to_become_available(device): + """Wait for a disk device to become available. + + Waits for a disk device to become available for use by + waiting until all process locks on the device have been + released. + + Timeout and iteration settings come from the configuration + options used by the in-library disk_partitioner: + ``check_device_interval`` and ``check_device_max_retries``. + + :params device: The path to the device. + :raises: IronicException If the disk fails to become + available. + """ + pids = [''] + stderr = [''] + interval = CONF.disk_partitioner.check_device_interval + max_retries = CONF.disk_partitioner.check_device_max_retries + + def _wait_for_disk(): + # A regex is likely overkill here, but variations in fuser + # means we should likely use it. + fuser_pids_re = re.compile(r'\d+') + + # There are 'psmisc' and 'busybox' versions of the 'fuser' program. The + # 'fuser' programs differ in how they output data to stderr. The + # busybox version does not output the filename to stderr, while the + # standard 'psmisc' version does output the filename to stderr. How + # they output to stdout is almost identical in that only the PIDs are + # output to stdout, with the 'psmisc' version adding a leading space + # character to the list of PIDs. + try: + # NOTE(ifarkas): fuser returns a non-zero return code if none of + # the specified files is accessed. + # NOTE(TheJulia): fuser does not report LVM devices as in use + # unless the LVM device-mapper device is the + # device that is directly polled. + # NOTE(TheJulia): The -m flag allows fuser to reveal data about + # mounted filesystems, which should be considered + # busy/locked. That being said, it is not used + # because busybox fuser has a different behavior. + # NOTE(TheJuia): fuser outputs a list of found PIDs to stdout. + # All other text is returned via stderr, and the + # output to a terminal is merged as a result. + out, err = utils.execute('fuser', device, check_exit_code=[0, 1]) + + if not out and not err: + return True + + stderr[0] = err + # NOTE: findall() returns a list of matches, or an empty list if no + # matches + pids[0] = fuser_pids_re.findall(out) + + except processutils.ProcessExecutionError as exc: + LOG.warning('Failed to check the device %(device)s with fuser:' + ' %(err)s', {'device': device, 'err': exc}) + return False + + retry = tenacity.retry( + retry=tenacity.retry_if_result(lambda r: not r), + stop=tenacity.stop_after_attempt(max_retries), + wait=tenacity.wait_fixed(interval), + reraise=True) + try: + retry(_wait_for_disk)() + except tenacity.RetryError: + if pids[0]: + raise exception.IronicException( + _('Processes with the following PIDs are holding ' + 'device %(device)s: %(pids)s. ' + 'Timed out waiting for completion.') + % {'device': device, 'pids': ', '.join(pids[0])}) + else: + raise exception.IronicException( + _('Fuser exited with "%(fuser_err)s" while checking ' + 'locks for device %(device)s. Timed out waiting for ' + 'completion.') + % {'device': device, 'fuser_err': stderr[0]}) diff --git a/ironic_python_agent/efi_utils.py b/ironic_python_agent/efi_utils.py index 077f7f86b..5be8dbc43 100644 --- a/ironic_python_agent/efi_utils.py +++ b/ironic_python_agent/efi_utils.py @@ -15,10 +15,10 @@ import re import sys import tempfile -from ironic_lib import disk_utils from oslo_concurrency import processutils from oslo_log import log +from ironic_python_agent import disk_utils from ironic_python_agent import errors from ironic_python_agent import hardware from ironic_python_agent import partition_utils diff --git a/ironic_python_agent/extensions/standby.py b/ironic_python_agent/extensions/standby.py index 9d77c73aa..7a365aa0f 100644 --- a/ironic_python_agent/extensions/standby.py +++ b/ironic_python_agent/extensions/standby.py @@ -19,13 +19,14 @@ import tempfile import time from urllib import parse as urlparse -from ironic_lib import disk_utils from ironic_lib import exception +from ironic_lib import qemu_img from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log import requests +from ironic_python_agent import disk_utils from ironic_python_agent import errors from ironic_python_agent.extensions import base from ironic_python_agent import hardware @@ -349,9 +350,9 @@ def _write_whole_disk_image(image, image_info, device): image, device] LOG.info('Writing image with command: %s', ' '.join(command)) try: - disk_utils.convert_image(image, device, out_format='host_device', - cache='directsync', out_of_order=True, - sparse_size='0') + qemu_img.convert_image(image, device, out_format='host_device', + cache='directsync', out_of_order=True, + sparse_size='0') except processutils.ProcessExecutionError as e: raise errors.ImageWriteError(device, e.exit_code, e.stdout, e.stderr) @@ -750,17 +751,7 @@ def _validate_partitioning(device): Check if after writing the image to disk we have a valid partition table by trying to read it. This will fail if the disk is junk. """ - try: - # Ensure we re-read the partition table before we try to list - # partitions - utils.execute('partprobe', device, - attempts=CONF.disk_utils.partprobe_attempts) - except (processutils.UnknownArgumentError, - processutils.ProcessExecutionError, OSError) as e: - LOG.warning("Unable to probe for partitions on device %(device)s " - "after writing the image, the partitioning table may " - "be broken. Error: %(error)s", - {'device': device, 'error': e}) + disk_utils.partprobe(device) try: nparts = len(disk_utils.list_partitions(device)) diff --git a/ironic_python_agent/hardware.py b/ironic_python_agent/hardware.py index 8eaa5823e..a5f37a48f 100644 --- a/ironic_python_agent/hardware.py +++ b/ironic_python_agent/hardware.py @@ -29,7 +29,6 @@ import stat import string import time -from ironic_lib import disk_utils from ironic_lib import utils as il_utils from oslo_concurrency import processutils from oslo_config import cfg @@ -41,6 +40,7 @@ import stevedore import yaml from ironic_python_agent import burnin +from ironic_python_agent import disk_utils from ironic_python_agent import encoding from ironic_python_agent import errors from ironic_python_agent.extensions import base as ext_base @@ -102,21 +102,6 @@ def _get_device_info(dev, devclass, field): {'field': field, 'dev': dev, 'class': devclass}) -def _udev_settle(): - """Wait for the udev event queue to settle. - - Wait for the udev event queue to settle to make sure all devices - are detected once the machine boots up. - - """ - try: - il_utils.execute('udevadm', 'settle') - except processutils.ProcessExecutionError as e: - LOG.warning('Something went wrong when waiting for udev ' - 'to settle. Error: %s', e) - return - - def _load_ipmi_modules(): """Load kernel modules required for IPMI interaction. @@ -508,7 +493,7 @@ def list_all_block_devices(block_type='disk', check_multipath = not ignore_multipath and get_multipath_status() - _udev_settle() + disk_utils.udev_settle() # map device names to /dev/disk/by-path symbolic links that points to it diff --git a/ironic_python_agent/inject_files.py b/ironic_python_agent/inject_files.py index 262ec1129..93f623584 100644 --- a/ironic_python_agent/inject_files.py +++ b/ironic_python_agent/inject_files.py @@ -16,12 +16,12 @@ import base64 import contextlib import os -from ironic_lib import disk_utils from ironic_lib import utils as ironic_utils from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log +from ironic_python_agent import disk_utils from ironic_python_agent import errors from ironic_python_agent import hardware from ironic_python_agent import utils diff --git a/ironic_python_agent/partition_utils.py b/ironic_python_agent/partition_utils.py index 77e5357f1..f410ef62f 100644 --- a/ironic_python_agent/partition_utils.py +++ b/ironic_python_agent/partition_utils.py @@ -26,7 +26,6 @@ import shutil import stat import tempfile -from ironic_lib import disk_utils from ironic_lib import exception from ironic_lib import utils from oslo_concurrency import processutils @@ -37,6 +36,7 @@ from oslo_utils import units from oslo_utils import uuidutils import requests +from ironic_python_agent import disk_utils from ironic_python_agent import errors from ironic_python_agent import hardware from ironic_python_agent import utils as ipa_utils diff --git a/ironic_python_agent/raid_utils.py b/ironic_python_agent/raid_utils.py index 8f3be1858..d4000eb1b 100644 --- a/ironic_python_agent/raid_utils.py +++ b/ironic_python_agent/raid_utils.py @@ -14,11 +14,11 @@ import copy import re import shlex -from ironic_lib import disk_utils from ironic_lib import utils as il_utils from oslo_concurrency import processutils from oslo_log import log as logging +from ironic_python_agent import disk_utils from ironic_python_agent import errors from ironic_python_agent import utils diff --git a/ironic_python_agent/tests/unit/extensions/test_image.py b/ironic_python_agent/tests/unit/extensions/test_image.py index 7010bd369..246c461e7 100644 --- a/ironic_python_agent/tests/unit/extensions/test_image.py +++ b/ironic_python_agent/tests/unit/extensions/test_image.py @@ -18,10 +18,10 @@ import shutil import tempfile from unittest import mock -from ironic_lib import disk_utils from ironic_lib import utils as ilib_utils from oslo_concurrency import processutils +from ironic_python_agent import disk_utils from ironic_python_agent import efi_utils from ironic_python_agent import errors from ironic_python_agent.extensions import image diff --git a/ironic_python_agent/tests/unit/extensions/test_standby.py b/ironic_python_agent/tests/unit/extensions/test_standby.py index bc5238c31..a0b161036 100644 --- a/ironic_python_agent/tests/unit/extensions/test_standby.py +++ b/ironic_python_agent/tests/unit/extensions/test_standby.py @@ -279,11 +279,14 @@ class TestStandbyExtension(base.IronicAgentTest): None, image_info['id']) - @mock.patch('ironic_lib.disk_utils.fix_gpt_partition', autospec=True) - @mock.patch('ironic_lib.disk_utils.trigger_device_rescan', autospec=True) - @mock.patch('ironic_lib.disk_utils.convert_image', autospec=True) - @mock.patch('ironic_lib.disk_utils.udev_settle', autospec=True) - @mock.patch('ironic_lib.disk_utils.destroy_disk_metadata', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.fix_gpt_partition', + autospec=True) + @mock.patch('ironic_python_agent.disk_utils.trigger_device_rescan', + autospec=True) + @mock.patch('ironic_lib.qemu_img.convert_image', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.udev_settle', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.destroy_disk_metadata', + autospec=True) def test_write_image(self, wipe_mock, udev_mock, convert_mock, rescan_mock, fix_gpt_mock): image_info = _build_fake_image_info() @@ -302,11 +305,14 @@ class TestStandbyExtension(base.IronicAgentTest): rescan_mock.assert_called_once_with(device) fix_gpt_mock.assert_called_once_with(device, node_uuid=None) - @mock.patch('ironic_lib.disk_utils.fix_gpt_partition', autospec=True) - @mock.patch('ironic_lib.disk_utils.trigger_device_rescan', autospec=True) - @mock.patch('ironic_lib.disk_utils.convert_image', autospec=True) - @mock.patch('ironic_lib.disk_utils.udev_settle', autospec=True) - @mock.patch('ironic_lib.disk_utils.destroy_disk_metadata', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.fix_gpt_partition', + autospec=True) + @mock.patch('ironic_python_agent.disk_utils.trigger_device_rescan', + autospec=True) + @mock.patch('ironic_lib.qemu_img.convert_image', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.udev_settle', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.destroy_disk_metadata', + autospec=True) def test_write_image_gpt_fails(self, wipe_mock, udev_mock, convert_mock, rescan_mock, fix_gpt_mock): image_info = _build_fake_image_info() @@ -315,9 +321,10 @@ class TestStandbyExtension(base.IronicAgentTest): fix_gpt_mock.side_effect = exception.InstanceDeployFailure standby._write_image(image_info, device) - @mock.patch('ironic_lib.disk_utils.convert_image', autospec=True) - @mock.patch('ironic_lib.disk_utils.udev_settle', autospec=True) - @mock.patch('ironic_lib.disk_utils.destroy_disk_metadata', autospec=True) + @mock.patch('ironic_lib.qemu_img.convert_image', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.udev_settle', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.destroy_disk_metadata', + autospec=True) def test_write_image_fails(self, wipe_mock, udev_mock, convert_mock): image_info = _build_fake_image_info() device = '/dev/sda' @@ -332,7 +339,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True) @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) - @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.get_image_mb', autospec=True) @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) def test_write_partition_image_exception(self, work_on_disk_mock, image_mb_mock, @@ -376,7 +383,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True) @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) - @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.get_image_mb', autospec=True) @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) def test_write_partition_image_no_node_uuid(self, work_on_disk_mock, image_mb_mock, @@ -423,7 +430,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True) @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) - @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.get_image_mb', autospec=True) @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) def test_write_partition_image_exception_image_mb(self, work_on_disk_mock, @@ -450,7 +457,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) - @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.get_image_mb', autospec=True) def test_write_partition_image(self, image_mb_mock, work_on_disk_mock, execute_mock, open_mock, dispatch_mock): image_info = _build_fake_partition_image_info() @@ -837,11 +844,10 @@ class TestStandbyExtension(base.IronicAgentTest): standby.ImageDownload, image_info) - @mock.patch('ironic_lib.disk_utils.get_disk_identifier', + @mock.patch('ironic_python_agent.disk_utils.get_disk_identifier', lambda dev: 'ROOT') - @mock.patch('ironic_python_agent.utils.execute', - autospec=True) - @mock.patch('ironic_lib.disk_utils.list_partitions', + @mock.patch('ironic_lib.utils.execute', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.list_partitions', autospec=True) @mock.patch.object(partition_utils, 'create_config_drive_partition', autospec=True) @@ -891,8 +897,8 @@ class TestStandbyExtension(base.IronicAgentTest): self.assertEqual({'root uuid': 'ROOT'}, self.agent_extension.partition_uuids) - @mock.patch('ironic_python_agent.utils.execute', autospec=True) - @mock.patch('ironic_lib.disk_utils.list_partitions', + @mock.patch('ironic_lib.utils.execute', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.list_partitions', autospec=True) @mock.patch.object(partition_utils, 'create_config_drive_partition', autospec=True) @@ -962,12 +968,12 @@ class TestStandbyExtension(base.IronicAgentTest): self.assertEqual({'root uuid': 'root_uuid'}, self.agent_extension.partition_uuids) - @mock.patch('ironic_lib.disk_utils.get_disk_identifier', + @mock.patch('ironic_python_agent.disk_utils.get_disk_identifier', lambda dev: 'ROOT') - @mock.patch('ironic_python_agent.utils.execute', autospec=True) + @mock.patch('ironic_lib.utils.execute', autospec=True) @mock.patch.object(partition_utils, 'create_config_drive_partition', autospec=True) - @mock.patch('ironic_lib.disk_utils.list_partitions', + @mock.patch('ironic_python_agent.disk_utils.list_partitions', autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', autospec=True) @@ -1007,12 +1013,12 @@ class TestStandbyExtension(base.IronicAgentTest): 'root_uuid=ROOT').format(image_info['id'], 'manager') self.assertEqual(cmd_result, async_result.command_result['result']) - @mock.patch('ironic_lib.disk_utils.get_disk_identifier', + @mock.patch('ironic_python_agent.disk_utils.get_disk_identifier', lambda dev: 'ROOT') @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) @mock.patch.object(partition_utils, 'create_config_drive_partition', autospec=True) - @mock.patch('ironic_lib.disk_utils.list_partitions', + @mock.patch('ironic_python_agent.disk_utils.list_partitions', autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', autospec=True) @@ -1054,11 +1060,11 @@ class TestStandbyExtension(base.IronicAgentTest): self.assertFalse(configdrive_copy_mock.called) self.assertEqual('FAILED', async_result.command_status) - @mock.patch('ironic_lib.disk_utils.get_disk_identifier', + @mock.patch('ironic_python_agent.disk_utils.get_disk_identifier', side_effect=OSError, autospec=True) - @mock.patch('ironic_python_agent.utils.execute', + @mock.patch('ironic_lib.utils.execute', autospec=True) - @mock.patch('ironic_lib.disk_utils.list_partitions', + @mock.patch('ironic_python_agent.disk_utils.list_partitions', autospec=True) @mock.patch.object(partition_utils, 'create_config_drive_partition', autospec=True) @@ -1108,10 +1114,10 @@ class TestStandbyExtension(base.IronicAgentTest): attempts=mock.ANY) self.assertEqual({}, self.agent_extension.partition_uuids) - @mock.patch('ironic_python_agent.utils.execute', mock.Mock()) - @mock.patch('ironic_lib.disk_utils.list_partitions', + @mock.patch('ironic_lib.utils.execute', mock.Mock()) + @mock.patch('ironic_python_agent.disk_utils.list_partitions', lambda _dev: [mock.Mock()]) - @mock.patch('ironic_lib.disk_utils.get_disk_identifier', + @mock.patch('ironic_python_agent.disk_utils.get_disk_identifier', lambda dev: 'ROOT') @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) @mock.patch.object(partition_utils, 'create_config_drive_partition', @@ -1346,8 +1352,9 @@ class TestStandbyExtension(base.IronicAgentTest): 'configdrive_data') @mock.patch('ironic_python_agent.extensions.standby.LOG', autospec=True) - @mock.patch('ironic_lib.disk_utils.block_uuid', autospec=True) - @mock.patch('ironic_lib.disk_utils.fix_gpt_partition', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.block_uuid', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.fix_gpt_partition', + autospec=True) @mock.patch('hashlib.new', autospec=True) @mock.patch('builtins.open', autospec=True) @mock.patch('requests.get', autospec=True) @@ -1444,7 +1451,8 @@ class TestStandbyExtension(base.IronicAgentTest): mock.call(b'some')] file_mock.write.assert_has_calls(write_calls) - @mock.patch('ironic_lib.disk_utils.fix_gpt_partition', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.fix_gpt_partition', + autospec=True) @mock.patch('hashlib.new', autospec=True) @mock.patch('builtins.open', autospec=True) @mock.patch('requests.get', autospec=True) @@ -1570,7 +1578,7 @@ class TestStandbyExtension(base.IronicAgentTest): @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True) @mock.patch('builtins.open', autospec=True) @mock.patch('ironic_python_agent.utils.execute', autospec=True) - @mock.patch('ironic_lib.disk_utils.get_image_mb', autospec=True) + @mock.patch('ironic_python_agent.disk_utils.get_image_mb', autospec=True) @mock.patch.object(partition_utils, 'work_on_disk', autospec=True) def test_write_partition_image_no_node_uuid_uefi( self, work_on_disk_mock, diff --git a/ironic_python_agent/tests/unit/test_disk_partitioner.py b/ironic_python_agent/tests/unit/test_disk_partitioner.py new file mode 100644 index 000000000..bdf0c0de7 --- /dev/null +++ b/ironic_python_agent/tests/unit/test_disk_partitioner.py @@ -0,0 +1,202 @@ +# Copyright 2014 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock + +from ironic_lib import exception +from ironic_lib.tests import base +from ironic_lib import utils + +from ironic_python_agent import disk_partitioner + + +CONF = disk_partitioner.CONF + + +class DiskPartitionerTestCase(base.IronicLibTestCase): + + def test_add_partition(self): + dp = disk_partitioner.DiskPartitioner('/dev/fake') + dp.add_partition(1024) + dp.add_partition(512, fs_type='linux-swap') + dp.add_partition(2048, boot_flag='boot') + dp.add_partition(2048, boot_flag='bios_grub') + expected = [(1, {'boot_flag': None, + 'extra_flags': None, + 'fs_type': '', + 'type': 'primary', + 'size': 1024}), + (2, {'boot_flag': None, + 'extra_flags': None, + 'fs_type': 'linux-swap', + 'type': 'primary', + 'size': 512}), + (3, {'boot_flag': 'boot', + 'extra_flags': None, + 'fs_type': '', + 'type': 'primary', + 'size': 2048}), + (4, {'boot_flag': 'bios_grub', + 'extra_flags': None, + 'fs_type': '', + 'type': 'primary', + 'size': 2048})] + partitions = [(n, p) for n, p in dp.get_partitions()] + self.assertEqual(4, len(partitions)) + self.assertEqual(expected, partitions) + + @mock.patch.object(disk_partitioner.DiskPartitioner, '_exec', + autospec=True) + @mock.patch.object(utils, 'execute', autospec=True) + def test_commit(self, mock_utils_exc, mock_disk_partitioner_exec): + dp = disk_partitioner.DiskPartitioner('/dev/fake') + fake_parts = [(1, {'boot_flag': None, + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1}), + (2, {'boot_flag': 'boot', + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1}), + (3, {'boot_flag': 'bios_grub', + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1}), + (4, {'boot_flag': 'boot', + 'extra_flags': ['prep', 'fake-flag'], + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1})] + with mock.patch.object(dp, 'get_partitions', autospec=True) as mock_gp: + mock_gp.return_value = fake_parts + mock_utils_exc.return_value = ('', '') + dp.commit() + + mock_disk_partitioner_exec.assert_called_once_with( + mock.ANY, 'mklabel', 'msdos', + 'mkpart', 'fake-type', 'fake-fs-type', '1', '2', + 'mkpart', 'fake-type', 'fake-fs-type', '2', '3', + 'set', '2', 'boot', 'on', + 'mkpart', 'fake-type', 'fake-fs-type', '3', '4', + 'set', '3', 'bios_grub', 'on', + 'mkpart', 'fake-type', 'fake-fs-type', '4', '5', + 'set', '4', 'boot', 'on', 'set', '4', 'prep', 'on', + 'set', '4', 'fake-flag', 'on') + mock_utils_exc.assert_called_once_with( + 'fuser', '/dev/fake', check_exit_code=[0, 1]) + + @mock.patch.object(disk_partitioner.DiskPartitioner, '_exec', + autospec=True) + @mock.patch.object(utils, 'execute', autospec=True) + def test_commit_with_device_is_busy_once(self, mock_utils_exc, + mock_disk_partitioner_exec): + CONF.set_override('check_device_interval', 0, group='disk_partitioner') + dp = disk_partitioner.DiskPartitioner('/dev/fake') + fake_parts = [(1, {'boot_flag': None, + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1}), + (2, {'boot_flag': 'boot', + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1})] + # Test as if the 'psmisc' version of 'fuser' which has stderr output + fuser_outputs = iter([(" 10000 10001", '/dev/fake:\n'), ('', '')]) + + with mock.patch.object(dp, 'get_partitions', autospec=True) as mock_gp: + mock_gp.return_value = fake_parts + mock_utils_exc.side_effect = fuser_outputs + dp.commit() + + mock_disk_partitioner_exec.assert_called_once_with( + mock.ANY, 'mklabel', 'msdos', + 'mkpart', 'fake-type', 'fake-fs-type', '1', '2', + 'mkpart', 'fake-type', 'fake-fs-type', '2', '3', + 'set', '2', 'boot', 'on') + mock_utils_exc.assert_called_with( + 'fuser', '/dev/fake', check_exit_code=[0, 1]) + self.assertEqual(2, mock_utils_exc.call_count) + + @mock.patch.object(disk_partitioner.DiskPartitioner, '_exec', + autospec=True) + @mock.patch.object(utils, 'execute', autospec=True) + def test_commit_with_device_is_always_busy(self, mock_utils_exc, + mock_disk_partitioner_exec): + CONF.set_override('check_device_interval', 0, group='disk_partitioner') + dp = disk_partitioner.DiskPartitioner('/dev/fake') + fake_parts = [(1, {'boot_flag': None, + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1}), + (2, {'boot_flag': 'boot', + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1})] + + with mock.patch.object(dp, 'get_partitions', autospec=True) as mock_gp: + mock_gp.return_value = fake_parts + # Test as if the 'busybox' version of 'fuser' which does not have + # stderr output + mock_utils_exc.return_value = ("10000 10001", '') + self.assertRaises(exception.InstanceDeployFailure, dp.commit) + + mock_disk_partitioner_exec.assert_called_once_with( + mock.ANY, 'mklabel', 'msdos', + 'mkpart', 'fake-type', 'fake-fs-type', '1', '2', + 'mkpart', 'fake-type', 'fake-fs-type', '2', '3', + 'set', '2', 'boot', 'on') + mock_utils_exc.assert_called_with( + 'fuser', '/dev/fake', check_exit_code=[0, 1]) + self.assertEqual(20, mock_utils_exc.call_count) + + @mock.patch.object(disk_partitioner.DiskPartitioner, '_exec', + autospec=True) + @mock.patch.object(utils, 'execute', autospec=True) + def test_commit_with_device_disconnected(self, mock_utils_exc, + mock_disk_partitioner_exec): + CONF.set_override('check_device_interval', 0, group='disk_partitioner') + dp = disk_partitioner.DiskPartitioner('/dev/fake') + fake_parts = [(1, {'boot_flag': None, + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1}), + (2, {'boot_flag': 'boot', + 'extra_flags': None, + 'fs_type': 'fake-fs-type', + 'type': 'fake-type', + 'size': 1})] + + with mock.patch.object(dp, 'get_partitions', autospec=True) as mock_gp: + mock_gp.return_value = fake_parts + mock_utils_exc.return_value = ('', "Specified filename /dev/fake" + " does not exist.") + self.assertRaises(exception.InstanceDeployFailure, dp.commit) + + mock_disk_partitioner_exec.assert_called_once_with( + mock.ANY, 'mklabel', 'msdos', + 'mkpart', 'fake-type', 'fake-fs-type', '1', '2', + 'mkpart', 'fake-type', 'fake-fs-type', '2', '3', + 'set', '2', 'boot', 'on') + mock_utils_exc.assert_called_with( + 'fuser', '/dev/fake', check_exit_code=[0, 1]) + self.assertEqual(20, mock_utils_exc.call_count) diff --git a/ironic_python_agent/tests/unit/test_disk_utils.py b/ironic_python_agent/tests/unit/test_disk_utils.py new file mode 100644 index 000000000..4c0275df2 --- /dev/null +++ b/ironic_python_agent/tests/unit/test_disk_utils.py @@ -0,0 +1,927 @@ +# Copyright 2014 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import stat +from unittest import mock + +from ironic_lib import exception +from ironic_lib import qemu_img +from ironic_lib.tests import base +from ironic_lib import utils +from oslo_concurrency import processutils +from oslo_config import cfg + +from ironic_python_agent import disk_utils + +CONF = cfg.CONF + + +@mock.patch.object(utils, 'execute', autospec=True) +class ListPartitionsTestCase(base.IronicLibTestCase): + + def test_correct(self, execute_mock): + output = """ +BYT; +/dev/sda:500107862016B:scsi:512:4096:msdos:ATA HGST HTS725050A7:; +1:1.00MiB:501MiB:500MiB:ext4::boot; +2:501MiB:476940MiB:476439MiB:::; +""" + expected = [ + {'number': 1, 'start': 1, 'end': 501, 'size': 500, + 'filesystem': 'ext4', 'partition_name': '', 'flags': 'boot', + 'path': '/dev/fake1'}, + {'number': 2, 'start': 501, 'end': 476940, 'size': 476439, + 'filesystem': '', 'partition_name': '', 'flags': '', + 'path': '/dev/fake2'}, + ] + execute_mock.return_value = (output, '') + result = disk_utils.list_partitions('/dev/fake') + self.assertEqual(expected, result) + execute_mock.assert_called_once_with( + 'parted', '-s', '-m', '/dev/fake', 'unit', 'MiB', 'print', + use_standard_locale=True) + + @mock.patch.object(disk_utils.LOG, 'warning', autospec=True) + def test_incorrect(self, log_mock, execute_mock): + output = """ +BYT; +/dev/sda:500107862016B:scsi:512:4096:msdos:ATA HGST HTS725050A7:; +1:XX1076MiB:---:524MiB:ext4::boot; +""" + execute_mock.return_value = (output, '') + self.assertEqual([], disk_utils.list_partitions('/dev/fake')) + self.assertEqual(1, log_mock.call_count) + + def test_correct_gpt_nvme(self, execute_mock): + output = """ +BYT; +/dev/vda:40960MiB:virtblk:512:512:gpt:Virtio Block Device:; +2:1.00MiB:2.00MiB:1.00MiB::Bios partition:bios_grub; +1:4.00MiB:5407MiB:5403MiB:ext4:Root partition:; +3:5407MiB:5507MiB:100MiB:fat16:Boot partition:boot, esp; +""" + expected = [ + {'end': 2, 'number': 2, 'start': 1, 'flags': 'bios_grub', + 'filesystem': '', 'partition_name': 'Bios partition', 'size': 1, + 'path': '/dev/fake0p2'}, + {'end': 5407, 'number': 1, 'start': 4, 'flags': '', + 'filesystem': 'ext4', 'partition_name': 'Root partition', + 'size': 5403, 'path': '/dev/fake0p1'}, + {'end': 5507, 'number': 3, 'start': 5407, + 'flags': 'boot, esp', 'filesystem': 'fat16', + 'partition_name': 'Boot partition', 'size': 100, + 'path': '/dev/fake0p3'}, + ] + execute_mock.return_value = (output, '') + result = disk_utils.list_partitions('/dev/fake0') + self.assertEqual(expected, result) + execute_mock.assert_called_once_with( + 'parted', '-s', '-m', '/dev/fake0', 'unit', 'MiB', 'print', + use_standard_locale=True) + + @mock.patch.object(disk_utils.LOG, 'warning', autospec=True) + def test_incorrect_gpt(self, log_mock, execute_mock): + output = """ +BYT; +/dev/vda:40960MiB:virtblk:512:512:gpt:Virtio Block Device:; +2:XX1.00MiB:---:1.00MiB::primary:bios_grub; +""" + execute_mock.return_value = (output, '') + self.assertEqual([], disk_utils.list_partitions('/dev/fake')) + self.assertEqual(1, log_mock.call_count) + + +@mock.patch.object(utils, 'execute', autospec=True) +class MakePartitionsTestCase(base.IronicLibTestCase): + + def setUp(self): + super(MakePartitionsTestCase, self).setUp() + self.dev = 'fake-dev' + self.root_mb = 1024 + self.swap_mb = 512 + self.ephemeral_mb = 0 + self.configdrive_mb = 0 + self.node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + self.efi_size = CONF.disk_utils.efi_system_partition_size + self.bios_size = CONF.disk_utils.bios_boot_partition_size + + def _get_parted_cmd(self, dev, label=None): + if label is None: + label = 'msdos' + + return ['parted', '-a', 'optimal', '-s', dev, + '--', 'unit', 'MiB', 'mklabel', label] + + def _add_efi_sz(self, x): + return str(x + self.efi_size) + + def _add_bios_sz(self, x): + return str(x + self.bios_size) + + def _test_make_partitions(self, mock_exc, boot_option, boot_mode='bios', + disk_label=None, cpu_arch=""): + mock_exc.return_value = ('', '') + disk_utils.make_partitions(self.dev, self.root_mb, self.swap_mb, + self.ephemeral_mb, self.configdrive_mb, + self.node_uuid, boot_option=boot_option, + boot_mode=boot_mode, disk_label=disk_label, + cpu_arch=cpu_arch) + + if boot_option == "local" and boot_mode == "uefi": + expected_mkpart = ['mkpart', 'primary', 'fat32', '1', + self._add_efi_sz(1), + 'set', '1', 'boot', 'on', + 'mkpart', 'primary', 'linux-swap', + self._add_efi_sz(1), self._add_efi_sz(513), + 'mkpart', 'primary', '', self._add_efi_sz(513), + self._add_efi_sz(1537)] + else: + if boot_option == "local": + if disk_label == "gpt": + if cpu_arch.startswith('ppc64'): + expected_mkpart = ['mkpart', 'primary', '', '1', '9', + 'set', '1', 'prep', 'on', + 'mkpart', 'primary', 'linux-swap', + '9', '521', 'mkpart', 'primary', + '', '521', '1545'] + else: + expected_mkpart = ['mkpart', 'primary', '', '1', + self._add_bios_sz(1), + 'set', '1', 'bios_grub', 'on', + 'mkpart', 'primary', 'linux-swap', + self._add_bios_sz(1), + self._add_bios_sz(513), + 'mkpart', 'primary', '', + self._add_bios_sz(513), + self._add_bios_sz(1537)] + elif cpu_arch.startswith('ppc64'): + expected_mkpart = ['mkpart', 'primary', '', '1', '9', + 'set', '1', 'boot', 'on', + 'set', '1', 'prep', 'on', + 'mkpart', 'primary', 'linux-swap', + '9', '521', 'mkpart', 'primary', + '', '521', '1545'] + else: + expected_mkpart = ['mkpart', 'primary', 'linux-swap', '1', + '513', 'mkpart', 'primary', '', '513', + '1537', 'set', '2', 'boot', 'on'] + else: + expected_mkpart = ['mkpart', 'primary', 'linux-swap', '1', + '513', 'mkpart', 'primary', '', '513', + '1537'] + self.dev = 'fake-dev' + parted_cmd = (self._get_parted_cmd(self.dev, disk_label) + + expected_mkpart) + parted_call = mock.call(*parted_cmd, use_standard_locale=True) + fuser_cmd = ['fuser', 'fake-dev'] + fuser_call = mock.call(*fuser_cmd, check_exit_code=[0, 1]) + + sync_calls = [mock.call('sync'), + mock.call('udevadm', 'settle'), + mock.call('partprobe', self.dev, attempts=10), + mock.call('udevadm', 'settle'), + mock.call('sgdisk', '-v', self.dev)] + + mock_exc.assert_has_calls([parted_call, fuser_call] + sync_calls) + + def test_make_partitions(self, mock_exc): + self._test_make_partitions(mock_exc, boot_option="netboot") + + def test_make_partitions_local_boot(self, mock_exc): + self._test_make_partitions(mock_exc, boot_option="local") + + def test_make_partitions_local_boot_uefi(self, mock_exc): + self._test_make_partitions(mock_exc, boot_option="local", + boot_mode="uefi", disk_label="gpt") + + def test_make_partitions_local_boot_gpt_bios(self, mock_exc): + self._test_make_partitions(mock_exc, boot_option="local", + disk_label="gpt") + + def test_make_partitions_disk_label_gpt(self, mock_exc): + self._test_make_partitions(mock_exc, boot_option="netboot", + disk_label="gpt") + + def test_make_partitions_mbr_with_prep(self, mock_exc): + self._test_make_partitions(mock_exc, boot_option="local", + disk_label="msdos", cpu_arch="ppc64le") + + def test_make_partitions_gpt_with_prep(self, mock_exc): + self._test_make_partitions(mock_exc, boot_option="local", + disk_label="gpt", cpu_arch="ppc64le") + + def test_make_partitions_with_ephemeral(self, mock_exc): + self.ephemeral_mb = 2048 + expected_mkpart = ['mkpart', 'primary', '', '1', '2049', + 'mkpart', 'primary', 'linux-swap', '2049', '2561', + 'mkpart', 'primary', '', '2561', '3585'] + self.dev = 'fake-dev' + cmd = self._get_parted_cmd(self.dev) + expected_mkpart + mock_exc.return_value = ('', '') + disk_utils.make_partitions(self.dev, self.root_mb, self.swap_mb, + self.ephemeral_mb, self.configdrive_mb, + self.node_uuid) + + parted_call = mock.call(*cmd, use_standard_locale=True) + mock_exc.assert_has_calls([parted_call]) + + def test_make_partitions_with_iscsi_device(self, mock_exc): + self.ephemeral_mb = 2048 + expected_mkpart = ['mkpart', 'primary', '', '1', '2049', + 'mkpart', 'primary', 'linux-swap', '2049', '2561', + 'mkpart', 'primary', '', '2561', '3585'] + self.dev = '/dev/iqn.2008-10.org.openstack:%s.fake-9' % self.node_uuid + ep = '/dev/iqn.2008-10.org.openstack:%s.fake-9-part1' % self.node_uuid + swap = ('/dev/iqn.2008-10.org.openstack:%s.fake-9-part2' + % self.node_uuid) + root = ('/dev/iqn.2008-10.org.openstack:%s.fake-9-part3' + % self.node_uuid) + expected_result = {'ephemeral': ep, + 'swap': swap, + 'root': root} + cmd = self._get_parted_cmd(self.dev) + expected_mkpart + mock_exc.return_value = ('', '') + result = disk_utils.make_partitions( + self.dev, self.root_mb, self.swap_mb, self.ephemeral_mb, + self.configdrive_mb, self.node_uuid) + + parted_call = mock.call(*cmd, use_standard_locale=True) + mock_exc.assert_has_calls([parted_call]) + self.assertEqual(expected_result, result) + + def test_make_partitions_with_nvme_device(self, mock_exc): + self.ephemeral_mb = 2048 + expected_mkpart = ['mkpart', 'primary', '', '1', '2049', + 'mkpart', 'primary', 'linux-swap', '2049', '2561', + 'mkpart', 'primary', '', '2561', '3585'] + self.dev = '/dev/nvmefake-9' + ep = '/dev/nvmefake-9p1' + swap = '/dev/nvmefake-9p2' + root = '/dev/nvmefake-9p3' + expected_result = {'ephemeral': ep, + 'swap': swap, + 'root': root} + cmd = self._get_parted_cmd(self.dev) + expected_mkpart + mock_exc.return_value = ('', '') + result = disk_utils.make_partitions( + self.dev, self.root_mb, self.swap_mb, self.ephemeral_mb, + self.configdrive_mb, self.node_uuid) + + parted_call = mock.call(*cmd, use_standard_locale=True) + mock_exc.assert_has_calls([parted_call]) + self.assertEqual(expected_result, result) + + def test_make_partitions_with_local_device(self, mock_exc): + self.ephemeral_mb = 2048 + expected_mkpart = ['mkpart', 'primary', '', '1', '2049', + 'mkpart', 'primary', 'linux-swap', '2049', '2561', + 'mkpart', 'primary', '', '2561', '3585'] + self.dev = 'fake-dev' + expected_result = {'ephemeral': 'fake-dev1', + 'swap': 'fake-dev2', + 'root': 'fake-dev3'} + cmd = self._get_parted_cmd(self.dev) + expected_mkpart + mock_exc.return_value = ('', '') + result = disk_utils.make_partitions( + self.dev, self.root_mb, self.swap_mb, self.ephemeral_mb, + self.configdrive_mb, self.node_uuid) + + parted_call = mock.call(*cmd, use_standard_locale=True) + mock_exc.assert_has_calls([parted_call]) + self.assertEqual(expected_result, result) + + +@mock.patch.object(utils, 'execute', autospec=True) +class DestroyMetaDataTestCase(base.IronicLibTestCase): + + def setUp(self): + super(DestroyMetaDataTestCase, self).setUp() + self.dev = 'fake-dev' + self.node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + + def test_destroy_disk_metadata(self, mock_exec): + # Note(TheJulia): This list will get-reused, but only the second + # execution returning a string is needed for the test as otherwise + # command output is not used. + mock_exec.side_effect = iter([ + (None, None), + ('1024\n', None), + (None, None), + (None, None), + (None, None), + (None, None)]) + + expected_calls = [mock.call('wipefs', '--force', '--all', 'fake-dev', + use_standard_locale=True), + mock.call('blockdev', '--getsz', 'fake-dev'), + mock.call('dd', 'bs=512', 'if=/dev/zero', + 'of=fake-dev', 'count=33', 'oflag=direct', + use_standard_locale=True), + mock.call('dd', 'bs=512', 'if=/dev/zero', + 'of=fake-dev', 'count=33', 'oflag=direct', + 'seek=991', use_standard_locale=True), + mock.call('sgdisk', '-Z', 'fake-dev', + use_standard_locale=True), + mock.call('fuser', self.dev, check_exit_code=[0, 1])] + disk_utils.destroy_disk_metadata(self.dev, self.node_uuid) + mock_exec.assert_has_calls(expected_calls) + + def test_destroy_disk_metadata_wipefs_fail(self, mock_exec): + mock_exec.side_effect = processutils.ProcessExecutionError + + expected_call = [mock.call('wipefs', '--force', '--all', 'fake-dev', + use_standard_locale=True)] + self.assertRaises(processutils.ProcessExecutionError, + disk_utils.destroy_disk_metadata, + self.dev, + self.node_uuid) + mock_exec.assert_has_calls(expected_call) + + def test_destroy_disk_metadata_sgdisk_fail(self, mock_exec): + expected_calls = [mock.call('wipefs', '--force', '--all', 'fake-dev', + use_standard_locale=True), + mock.call('blockdev', '--getsz', 'fake-dev'), + mock.call('dd', 'bs=512', 'if=/dev/zero', + 'of=fake-dev', 'count=33', 'oflag=direct', + use_standard_locale=True), + mock.call('dd', 'bs=512', 'if=/dev/zero', + 'of=fake-dev', 'count=33', 'oflag=direct', + 'seek=991', use_standard_locale=True), + mock.call('sgdisk', '-Z', 'fake-dev', + use_standard_locale=True)] + mock_exec.side_effect = iter([ + (None, None), + ('1024\n', None), + (None, None), + (None, None), + processutils.ProcessExecutionError()]) + self.assertRaises(processutils.ProcessExecutionError, + disk_utils.destroy_disk_metadata, + self.dev, + self.node_uuid) + mock_exec.assert_has_calls(expected_calls) + + def test_destroy_disk_metadata_wipefs_not_support_force(self, mock_exec): + mock_exec.side_effect = iter([ + processutils.ProcessExecutionError(description='--force'), + (None, None), + ('1024\n', None), + (None, None), + (None, None), + (None, None), + (None, None)]) + + expected_call = [mock.call('wipefs', '--force', '--all', 'fake-dev', + use_standard_locale=True), + mock.call('wipefs', '--all', 'fake-dev', + use_standard_locale=True)] + disk_utils.destroy_disk_metadata(self.dev, self.node_uuid) + mock_exec.assert_has_calls(expected_call) + + def test_destroy_disk_metadata_ebr(self, mock_exec): + expected_calls = [mock.call('wipefs', '--force', '--all', 'fake-dev', + use_standard_locale=True), + mock.call('blockdev', '--getsz', 'fake-dev'), + mock.call('dd', 'bs=512', 'if=/dev/zero', + 'of=fake-dev', 'count=2', 'oflag=direct', + use_standard_locale=True), + mock.call('sgdisk', '-Z', 'fake-dev', + use_standard_locale=True)] + mock_exec.side_effect = iter([ + (None, None), + ('2\n', None), # an EBR is 2 sectors + (None, None), + (None, None), + (None, None), + (None, None)]) + disk_utils.destroy_disk_metadata(self.dev, self.node_uuid) + mock_exec.assert_has_calls(expected_calls) + + def test_destroy_disk_metadata_tiny_partition(self, mock_exec): + expected_calls = [mock.call('wipefs', '--force', '--all', 'fake-dev', + use_standard_locale=True), + mock.call('blockdev', '--getsz', 'fake-dev'), + mock.call('dd', 'bs=512', 'if=/dev/zero', + 'of=fake-dev', 'count=33', 'oflag=direct', + use_standard_locale=True), + mock.call('dd', 'bs=512', 'if=/dev/zero', + 'of=fake-dev', 'count=33', 'oflag=direct', + 'seek=9', use_standard_locale=True), + mock.call('sgdisk', '-Z', 'fake-dev', + use_standard_locale=True)] + mock_exec.side_effect = iter([ + (None, None), + ('42\n', None), + (None, None), + (None, None), + (None, None), + (None, None)]) + disk_utils.destroy_disk_metadata(self.dev, self.node_uuid) + mock_exec.assert_has_calls(expected_calls) + + +@mock.patch.object(utils, 'execute', autospec=True) +class GetDeviceBlockSizeTestCase(base.IronicLibTestCase): + + def setUp(self): + super(GetDeviceBlockSizeTestCase, self).setUp() + self.dev = 'fake-dev' + self.node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + + def test_get_dev_block_size(self, mock_exec): + mock_exec.return_value = ("64", "") + expected_call = [mock.call('blockdev', '--getsz', self.dev)] + disk_utils.get_dev_block_size(self.dev) + mock_exec.assert_has_calls(expected_call) + + +@mock.patch.object(disk_utils, 'dd', autospec=True) +@mock.patch.object(qemu_img, 'image_info', autospec=True) +@mock.patch.object(qemu_img, 'convert_image', autospec=True) +class PopulateImageTestCase(base.IronicLibTestCase): + + def test_populate_raw_image(self, mock_cg, mock_qinfo, mock_dd): + type(mock_qinfo.return_value).file_format = mock.PropertyMock( + return_value='raw') + disk_utils.populate_image('src', 'dst') + mock_dd.assert_called_once_with('src', 'dst', conv_flags=None) + self.assertFalse(mock_cg.called) + + def test_populate_raw_image_with_convert(self, mock_cg, mock_qinfo, + mock_dd): + type(mock_qinfo.return_value).file_format = mock.PropertyMock( + return_value='raw') + disk_utils.populate_image('src', 'dst', conv_flags='sparse') + mock_dd.assert_called_once_with('src', 'dst', conv_flags='sparse') + self.assertFalse(mock_cg.called) + + def test_populate_qcow2_image(self, mock_cg, mock_qinfo, mock_dd): + type(mock_qinfo.return_value).file_format = mock.PropertyMock( + return_value='qcow2') + disk_utils.populate_image('src', 'dst') + mock_cg.assert_called_once_with('src', 'dst', 'raw', True, + sparse_size='0') + self.assertFalse(mock_dd.called) + + +@mock.patch('time.sleep', lambda sec: None) +class OtherFunctionTestCase(base.IronicLibTestCase): + + @mock.patch.object(os, 'stat', autospec=True) + @mock.patch.object(stat, 'S_ISBLK', autospec=True) + def test_is_block_device_works(self, mock_is_blk, mock_os): + device = '/dev/disk/by-path/ip-1.2.3.4:5678-iscsi-iqn.fake-lun-9' + mock_is_blk.return_value = True + mock_os().st_mode = 10000 + self.assertTrue(disk_utils.is_block_device(device)) + mock_is_blk.assert_called_once_with(mock_os().st_mode) + + @mock.patch.object(os, 'stat', autospec=True) + def test_is_block_device_raises(self, mock_os): + device = '/dev/disk/by-path/ip-1.2.3.4:5678-iscsi-iqn.fake-lun-9' + mock_os.side_effect = OSError + self.assertRaises(exception.InstanceDeployFailure, + disk_utils.is_block_device, device) + mock_os.assert_has_calls([mock.call(device)] * 3) + + @mock.patch.object(os, 'stat', autospec=True) + def test_is_block_device_attempts(self, mock_os): + CONF.set_override('partition_detection_attempts', 2, + group='disk_utils') + device = '/dev/disk/by-path/ip-1.2.3.4:5678-iscsi-iqn.fake-lun-9' + mock_os.side_effect = OSError + self.assertRaises(exception.InstanceDeployFailure, + disk_utils.is_block_device, device) + mock_os.assert_has_calls([mock.call(device)] * 2) + + @mock.patch.object(os.path, 'getsize', autospec=True) + @mock.patch.object(qemu_img, 'image_info', autospec=True) + def test_get_image_mb(self, mock_qinfo, mock_getsize): + mb = 1024 * 1024 + + mock_getsize.return_value = 0 + type(mock_qinfo.return_value).virtual_size = mock.PropertyMock( + return_value=0) + self.assertEqual(0, disk_utils.get_image_mb('x', False)) + self.assertEqual(0, disk_utils.get_image_mb('x', True)) + mock_getsize.return_value = 1 + type(mock_qinfo.return_value).virtual_size = mock.PropertyMock( + return_value=1) + self.assertEqual(1, disk_utils.get_image_mb('x', False)) + self.assertEqual(1, disk_utils.get_image_mb('x', True)) + mock_getsize.return_value = mb + type(mock_qinfo.return_value).virtual_size = mock.PropertyMock( + return_value=mb) + self.assertEqual(1, disk_utils.get_image_mb('x', False)) + self.assertEqual(1, disk_utils.get_image_mb('x', True)) + mock_getsize.return_value = mb + 1 + type(mock_qinfo.return_value).virtual_size = mock.PropertyMock( + return_value=mb + 1) + self.assertEqual(2, disk_utils.get_image_mb('x', False)) + self.assertEqual(2, disk_utils.get_image_mb('x', True)) + + def _test_count_mbr_partitions(self, output, mock_execute): + mock_execute.return_value = (output, '') + out = disk_utils.count_mbr_partitions('/dev/fake') + mock_execute.assert_called_once_with('partprobe', '-d', '-s', + '/dev/fake', + use_standard_locale=True) + return out + + @mock.patch.object(utils, 'execute', autospec=True) + def test_count_mbr_partitions(self, mock_execute): + output = "/dev/fake: msdos partitions 1 2 3 <5 6>" + pp, lp = self._test_count_mbr_partitions(output, mock_execute) + self.assertEqual(3, pp) + self.assertEqual(2, lp) + + @mock.patch.object(utils, 'execute', autospec=True) + def test_count_mbr_partitions_no_logical_partitions(self, mock_execute): + output = "/dev/fake: msdos partitions 1 2" + pp, lp = self._test_count_mbr_partitions(output, mock_execute) + self.assertEqual(2, pp) + self.assertEqual(0, lp) + + @mock.patch.object(utils, 'execute', autospec=True) + def test_count_mbr_partitions_wrong_partition_table(self, mock_execute): + output = "/dev/fake: gpt partitions 1 2 3 4 5 6" + mock_execute.return_value = (output, '') + self.assertRaises(ValueError, disk_utils.count_mbr_partitions, + '/dev/fake') + mock_execute.assert_called_once_with('partprobe', '-d', '-s', + '/dev/fake', + use_standard_locale=True) + + @mock.patch.object(disk_utils, 'get_device_information', autospec=True) + def test_block_uuid(self, mock_get_device_info): + mock_get_device_info.return_value = {'UUID': '123', + 'PARTUUID': '123456'} + self.assertEqual('123', disk_utils.block_uuid('/dev/fake')) + mock_get_device_info.assert_called_once_with( + '/dev/fake', fields=['UUID', 'PARTUUID']) + + @mock.patch.object(disk_utils, 'get_device_information', autospec=True) + def test_block_uuid_fallback_to_uuid(self, mock_get_device_info): + mock_get_device_info.return_value = {'PARTUUID': '123456'} + self.assertEqual('123456', disk_utils.block_uuid('/dev/fake')) + mock_get_device_info.assert_called_once_with( + '/dev/fake', fields=['UUID', 'PARTUUID']) + + +@mock.patch.object(utils, 'execute', autospec=True) +class FixGptStructsTestCases(base.IronicLibTestCase): + + def setUp(self): + super(FixGptStructsTestCases, self).setUp() + self.dev = "/dev/fake" + self.config_part_label = "config-2" + self.node_uuid = "12345678-1234-1234-1234-1234567890abcxyz" + + def test_fix_gpt_structs_fix_required(self, mock_execute): + sgdisk_v_output = """ +Problem: The secondary header's self-pointer indicates that it doesn't reside +at the end of the disk. If you've added a disk to a RAID array, use the 'e' +option on the experts' menu to adjust the secondary header's and partition +table's locations. + +Identified 1 problems! +""" + mock_execute.return_value = (sgdisk_v_output, '') + execute_calls = [ + mock.call('sgdisk', '-v', '/dev/fake'), + mock.call('sgdisk', '-e', '/dev/fake') + ] + disk_utils._fix_gpt_structs('/dev/fake', self.node_uuid) + mock_execute.assert_has_calls(execute_calls) + + def test_fix_gpt_structs_fix_not_required(self, mock_execute): + mock_execute.return_value = ('', '') + + disk_utils._fix_gpt_structs('/dev/fake', self.node_uuid) + mock_execute.assert_called_once_with('sgdisk', '-v', '/dev/fake') + + @mock.patch.object(disk_utils.LOG, 'error', autospec=True) + def test_fix_gpt_structs_exc(self, mock_log, mock_execute): + mock_execute.side_effect = processutils.ProcessExecutionError + self.assertRaisesRegex(exception.InstanceDeployFailure, + 'Failed to fix GPT data structures on disk', + disk_utils._fix_gpt_structs, + self.dev, self.node_uuid) + mock_execute.assert_called_once_with('sgdisk', '-v', '/dev/fake') + self.assertEqual(1, mock_log.call_count) + + +@mock.patch.object(utils, 'execute', autospec=True) +class TriggerDeviceRescanTestCase(base.IronicLibTestCase): + def test_trigger(self, mock_execute): + self.assertTrue(disk_utils.trigger_device_rescan('/dev/fake')) + mock_execute.assert_has_calls([ + mock.call('sync'), + mock.call('udevadm', 'settle'), + mock.call('partprobe', '/dev/fake', attempts=10), + mock.call('udevadm', 'settle'), + mock.call('sgdisk', '-v', '/dev/fake'), + ]) + + def test_custom_attempts(self, mock_execute): + self.assertTrue( + disk_utils.trigger_device_rescan('/dev/fake', attempts=1)) + mock_execute.assert_has_calls([ + mock.call('sync'), + mock.call('udevadm', 'settle'), + mock.call('partprobe', '/dev/fake', attempts=1), + mock.call('udevadm', 'settle'), + mock.call('sgdisk', '-v', '/dev/fake'), + ]) + + def test_fails(self, mock_execute): + mock_execute.side_effect = [('', '')] * 4 + [ + processutils.ProcessExecutionError + ] + self.assertFalse(disk_utils.trigger_device_rescan('/dev/fake')) + mock_execute.assert_has_calls([ + mock.call('sync'), + mock.call('udevadm', 'settle'), + mock.call('partprobe', '/dev/fake', attempts=10), + mock.call('udevadm', 'settle'), + mock.call('sgdisk', '-v', '/dev/fake'), + ]) + + +BLKID_PROBE = (""" +/dev/disk/by-path/ip-10.1.0.52:3260-iscsi-iqn.2008-10.org.openstack: """ + """PTUUID="123456" PTTYPE="gpt" + """) + +LSBLK_NORMAL = ( + 'UUID="123" BLOCK_SIZE="512" TYPE="vfat" ' + 'PARTLABEL="EFI System Partition" PARTUUID="123456"' +) + + +@mock.patch.object(utils, 'execute', autospec=True) +class GetDeviceInformationTestCase(base.IronicLibTestCase): + + def test_normal(self, mock_execute): + mock_execute.return_value = LSBLK_NORMAL, "" + result = disk_utils.get_device_information('/dev/fake') + self.assertEqual( + {'UUID': '123', 'BLOCK_SIZE': '512', 'TYPE': 'vfat', + 'PARTLABEL': 'EFI System Partition', 'PARTUUID': '123456'}, + result + ) + mock_execute.assert_called_once_with( + 'lsblk', '/dev/fake', '--pairs', '--bytes', '--ascii', '--nodeps', + '--output-all', use_standard_locale=True) + + def test_fields(self, mock_execute): + mock_execute.return_value = LSBLK_NORMAL, "" + result = disk_utils.get_device_information('/dev/fake', + fields=['UUID', 'LABEL']) + # No filtering on our side, so returning all fake fields + self.assertEqual( + {'UUID': '123', 'BLOCK_SIZE': '512', 'TYPE': 'vfat', + 'PARTLABEL': 'EFI System Partition', 'PARTUUID': '123456'}, + result + ) + mock_execute.assert_called_once_with( + 'lsblk', '/dev/fake', '--pairs', '--bytes', '--ascii', '--nodeps', + '--output', 'UUID,LABEL', + use_standard_locale=True) + + def test_empty(self, mock_execute): + mock_execute.return_value = "\n", "" + result = disk_utils.get_device_information('/dev/fake') + self.assertEqual({}, result) + mock_execute.assert_called_once_with( + 'lsblk', '/dev/fake', '--pairs', '--bytes', '--ascii', '--nodeps', + '--output-all', use_standard_locale=True) + + +@mock.patch.object(utils, 'execute', autospec=True) +class GetPartitionTableTypeTestCase(base.IronicLibTestCase): + def test_gpt(self, mocked_execute): + self._test_by_type(mocked_execute, 'gpt', 'gpt') + + def test_msdos(self, mocked_execute): + self._test_by_type(mocked_execute, 'msdos', 'msdos') + + def test_unknown(self, mocked_execute): + self._test_by_type(mocked_execute, 'whatever', 'unknown') + + def _test_by_type(self, mocked_execute, table_type_output, + expected_table_type): + parted_ret = PARTED_OUTPUT_UNFORMATTED.format(table_type_output) + + mocked_execute.side_effect = [ + (parted_ret, None), + ] + + ret = disk_utils.get_partition_table_type('hello') + mocked_execute.assert_called_once_with( + 'parted', '--script', 'hello', '--', 'print', + use_standard_locale=True) + self.assertEqual(expected_table_type, ret) + + +PARTED_OUTPUT_UNFORMATTED = '''Model: whatever +Disk /dev/sda: 450GB +Sector size (logical/physical): 512B/512B +Partition Table: {} +Disk Flags: + +Number Start End Size File system Name Flags +14 1049kB 5243kB 4194kB bios_grub +15 5243kB 116MB 111MB fat32 boot, esp + 1 116MB 2361MB 2245MB ext4 +''' + + +@mock.patch.object(disk_utils, 'list_partitions', autospec=True) +@mock.patch.object(disk_utils, 'get_partition_table_type', autospec=True) +class FindEfiPartitionTestCase(base.IronicLibTestCase): + + def test_find_efi_partition(self, mocked_type, mocked_parts): + mocked_parts.return_value = [ + {'number': '1', 'flags': ''}, + {'number': '14', 'flags': 'bios_grub'}, + {'number': '15', 'flags': 'esp, boot'}, + ] + ret = disk_utils.find_efi_partition('/dev/sda') + self.assertEqual({'number': '15', 'flags': 'esp, boot'}, ret) + + def test_find_efi_partition_only_boot_flag_gpt(self, mocked_type, + mocked_parts): + mocked_type.return_value = 'gpt' + mocked_parts.return_value = [ + {'number': '1', 'flags': ''}, + {'number': '14', 'flags': 'bios_grub'}, + {'number': '15', 'flags': 'boot'}, + ] + ret = disk_utils.find_efi_partition('/dev/sda') + self.assertEqual({'number': '15', 'flags': 'boot'}, ret) + + def test_find_efi_partition_only_boot_flag_mbr(self, mocked_type, + mocked_parts): + mocked_type.return_value = 'msdos' + mocked_parts.return_value = [ + {'number': '1', 'flags': ''}, + {'number': '14', 'flags': 'bios_grub'}, + {'number': '15', 'flags': 'boot'}, + ] + self.assertIsNone(disk_utils.find_efi_partition('/dev/sda')) + + def test_find_efi_partition_not_found(self, mocked_type, mocked_parts): + mocked_parts.return_value = [ + {'number': '1', 'flags': ''}, + {'number': '14', 'flags': 'bios_grub'}, + ] + self.assertIsNone(disk_utils.find_efi_partition('/dev/sda')) + + +class WaitForDisk(base.IronicLibTestCase): + + def setUp(self): + super(WaitForDisk, self).setUp() + CONF.set_override('check_device_interval', .01, + group='disk_partitioner') + CONF.set_override('check_device_max_retries', 2, + group='disk_partitioner') + + @mock.patch.object(utils, 'execute', autospec=True) + def test_wait_for_disk_to_become_available(self, mock_exc): + mock_exc.return_value = ('', '') + disk_utils.wait_for_disk_to_become_available('fake-dev') + fuser_cmd = ['fuser', 'fake-dev'] + fuser_call = mock.call(*fuser_cmd, check_exit_code=[0, 1]) + self.assertEqual(1, mock_exc.call_count) + mock_exc.assert_has_calls([fuser_call]) + + @mock.patch.object(utils, 'execute', autospec=True, + side_effect=processutils.ProcessExecutionError( + stderr='fake')) + def test_wait_for_disk_to_become_available_no_fuser(self, mock_exc): + self.assertRaises(exception.IronicException, + disk_utils.wait_for_disk_to_become_available, + 'fake-dev') + fuser_cmd = ['fuser', 'fake-dev'] + fuser_call = mock.call(*fuser_cmd, check_exit_code=[0, 1]) + self.assertEqual(2, mock_exc.call_count) + mock_exc.assert_has_calls([fuser_call, fuser_call]) + + @mock.patch.object(utils, 'execute', autospec=True) + def test_wait_for_disk_to_become_available_device_in_use_psmisc( + self, mock_exc): + # Test that the device is not available. This version has the 'psmisc' + # version of 'fuser' values for stdout and stderr. + # NOTE(TheJulia): Looks like fuser returns the actual list of pids + # in the stdout output, where as all other text is returned in + # stderr. + # The 'psmisc' version has a leading space character in stdout. The + # filename is output to stderr + mock_exc.side_effect = [(' 1234 ', 'fake-dev: '), + (' 15503 3919 15510 15511', 'fake-dev:')] + expected_error = ('Processes with the following PIDs are ' + 'holding device fake-dev: 15503, 3919, 15510, ' + '15511. Timed out waiting for completion.') + self.assertRaisesRegex( + exception.IronicException, + expected_error, + disk_utils.wait_for_disk_to_become_available, + 'fake-dev') + fuser_cmd = ['fuser', 'fake-dev'] + fuser_call = mock.call(*fuser_cmd, check_exit_code=[0, 1]) + self.assertEqual(2, mock_exc.call_count) + mock_exc.assert_has_calls([fuser_call, fuser_call]) + + @mock.patch.object(utils, 'execute', autospec=True) + def test_wait_for_disk_to_become_available_device_in_use_busybox( + self, mock_exc): + # Test that the device is not available. This version has the 'busybox' + # version of 'fuser' values for stdout and stderr. + # NOTE(TheJulia): Looks like fuser returns the actual list of pids + # in the stdout output, where as all other text is returned in + # stderr. + # The 'busybox' version does not have a leading space character in + # stdout. Also nothing is output to stderr. + mock_exc.side_effect = [('1234', ''), + ('15503 3919 15510 15511', '')] + expected_error = ('Processes with the following PIDs are ' + 'holding device fake-dev: 15503, 3919, 15510, ' + '15511. Timed out waiting for completion.') + self.assertRaisesRegex( + exception.IronicException, + expected_error, + disk_utils.wait_for_disk_to_become_available, + 'fake-dev') + fuser_cmd = ['fuser', 'fake-dev'] + fuser_call = mock.call(*fuser_cmd, check_exit_code=[0, 1]) + self.assertEqual(2, mock_exc.call_count) + mock_exc.assert_has_calls([fuser_call, fuser_call]) + + @mock.patch.object(utils, 'execute', autospec=True) + def test_wait_for_disk_to_become_available_no_device(self, mock_exc): + # NOTE(TheJulia): Looks like fuser returns the actual list of pids + # in the stdout output, where as all other text is returned in + # stderr. + + mock_exc.return_value = ('', 'Specified filename /dev/fake ' + 'does not exist.') + expected_error = ('Fuser exited with "Specified filename ' + '/dev/fake does not exist." while checking ' + 'locks for device fake-dev. Timed out waiting ' + 'for completion.') + self.assertRaisesRegex( + exception.IronicException, + expected_error, + disk_utils.wait_for_disk_to_become_available, + 'fake-dev') + fuser_cmd = ['fuser', 'fake-dev'] + fuser_call = mock.call(*fuser_cmd, check_exit_code=[0, 1]) + self.assertEqual(2, mock_exc.call_count) + mock_exc.assert_has_calls([fuser_call, fuser_call]) + + @mock.patch.object(utils, 'execute', autospec=True) + def test_wait_for_disk_to_become_available_dev_becomes_avail_psmisc( + self, mock_exc): + # Test that initially device is not available but then becomes + # available. This version has the 'psmisc' version of 'fuser' values + # for stdout and stderr. + # The 'psmisc' version has a leading space character in stdout. The + # filename is output to stderr + mock_exc.side_effect = [(' 1234 ', 'fake-dev: '), + ('', '')] + disk_utils.wait_for_disk_to_become_available('fake-dev') + fuser_cmd = ['fuser', 'fake-dev'] + fuser_call = mock.call(*fuser_cmd, check_exit_code=[0, 1]) + self.assertEqual(2, mock_exc.call_count) + mock_exc.assert_has_calls([fuser_call, fuser_call]) + + @mock.patch.object(utils, 'execute', autospec=True) + def test_wait_for_disk_to_become_available_dev_becomes_avail_busybox( + self, mock_exc): + # Test that initially device is not available but then becomes + # available. This version has the 'busybox' version of 'fuser' values + # for stdout and stderr. + # The 'busybox' version does not have a leading space character in + # stdout. Also nothing is output to stderr. + mock_exc.side_effect = [('1234 5895', ''), + ('', '')] + disk_utils.wait_for_disk_to_become_available('fake-dev') + fuser_cmd = ['fuser', 'fake-dev'] + fuser_call = mock.call(*fuser_cmd, check_exit_code=[0, 1]) + self.assertEqual(2, mock_exc.call_count) + mock_exc.assert_has_calls([fuser_call, fuser_call]) diff --git a/ironic_python_agent/tests/unit/test_efi_utils.py b/ironic_python_agent/tests/unit/test_efi_utils.py index e78c0fd5a..935bee8ec 100644 --- a/ironic_python_agent/tests/unit/test_efi_utils.py +++ b/ironic_python_agent/tests/unit/test_efi_utils.py @@ -15,9 +15,9 @@ import shutil import tempfile from unittest import mock -from ironic_lib import disk_utils from oslo_concurrency import processutils +from ironic_python_agent import disk_utils from ironic_python_agent import efi_utils from ironic_python_agent import errors from ironic_python_agent import hardware diff --git a/ironic_python_agent/tests/unit/test_hardware.py b/ironic_python_agent/tests/unit/test_hardware.py index 377139b4f..293454a32 100644 --- a/ironic_python_agent/tests/unit/test_hardware.py +++ b/ironic_python_agent/tests/unit/test_hardware.py @@ -20,7 +20,6 @@ import stat import time from unittest import mock -from ironic_lib import disk_utils from ironic_lib import utils as il_utils import netifaces from oslo_concurrency import processutils @@ -29,6 +28,7 @@ from oslo_utils import units import pyudev from stevedore import extension +from ironic_python_agent import disk_utils from ironic_python_agent import errors from ironic_python_agent import hardware from ironic_python_agent import netutils @@ -117,7 +117,7 @@ class TestHardwareManagerLoading(base.IronicAgentTest): ]) -@mock.patch.object(hardware, '_udev_settle', lambda *_: None) +@mock.patch.object(disk_utils, 'udev_settle', lambda *_: None) class TestGenericHardwareManager(base.IronicAgentTest): def setUp(self): super(TestGenericHardwareManager, self).setUp() @@ -5151,7 +5151,7 @@ class TestModuleFunctions(base.IronicAgentTest): @mock.patch.object(os, 'readlink', autospec=True) @mock.patch.object(hardware, '_get_device_info', lambda x, y, z: 'FooTastic') - @mock.patch.object(hardware, '_udev_settle', autospec=True) + @mock.patch.object(disk_utils, 'udev_settle', autospec=True) @mock.patch.object(hardware.pyudev.Devices, "from_device_file", autospec=False) def test_list_all_block_devices_success(self, mocked_fromdevfile, @@ -5185,7 +5185,7 @@ class TestModuleFunctions(base.IronicAgentTest): @mock.patch.object(os, 'readlink', autospec=True) @mock.patch.object(hardware, '_get_device_info', lambda x, y, z: 'FooTastic') - @mock.patch.object(hardware, '_udev_settle', autospec=True) + @mock.patch.object(disk_utils, 'udev_settle', autospec=True) @mock.patch.object(hardware.pyudev.Devices, "from_device_file", autospec=False) def test_list_all_block_devices_success_raid(self, mocked_fromdevfile, @@ -5236,7 +5236,7 @@ class TestModuleFunctions(base.IronicAgentTest): @mock.patch.object(os, 'readlink', autospec=True) @mock.patch.object(hardware, '_get_device_info', lambda x, y, z: 'FooTastic') - @mock.patch.object(hardware, '_udev_settle', autospec=True) + @mock.patch.object(disk_utils, 'udev_settle', autospec=True) @mock.patch.object(hardware.pyudev.Devices, "from_device_file", autospec=False) def test_list_all_block_devices_partuuid_success( @@ -5269,7 +5269,7 @@ class TestModuleFunctions(base.IronicAgentTest): @mock.patch.object(hardware, 'get_multipath_status', autospec=True) @mock.patch.object(hardware, '_get_device_info', lambda x, y: "FooTastic") - @mock.patch.object(hardware, '_udev_settle', autospec=True) + @mock.patch.object(disk_utils, 'udev_settle', autospec=True) def test_list_all_block_devices_wrong_block_type(self, mocked_udev, mock_mpath_enabled, mocked_execute): @@ -5285,7 +5285,7 @@ class TestModuleFunctions(base.IronicAgentTest): mocked_udev.assert_called_once_with() @mock.patch.object(hardware, 'get_multipath_status', autospec=True) - @mock.patch.object(hardware, '_udev_settle', autospec=True) + @mock.patch.object(disk_utils, 'udev_settle', autospec=True) def test_list_all_block_devices_missing(self, mocked_udev, mocked_mpath, mocked_execute): @@ -5306,10 +5306,6 @@ class TestModuleFunctions(base.IronicAgentTest): mocked_udev.assert_called_once_with() mocked_execute.assert_has_calls(expected_calls) - def test__udev_settle(self, mocked_execute): - hardware._udev_settle() - mocked_execute.assert_called_once_with('udevadm', 'settle') - def test__check_for_iscsi(self, mocked_execute): hardware._check_for_iscsi() mocked_execute.assert_has_calls([ diff --git a/ironic_python_agent/tests/unit/test_inject_files.py b/ironic_python_agent/tests/unit/test_inject_files.py index 30da3963f..f359ce519 100644 --- a/ironic_python_agent/tests/unit/test_inject_files.py +++ b/ironic_python_agent/tests/unit/test_inject_files.py @@ -22,7 +22,7 @@ from ironic_python_agent.tests.unit import base @mock.patch('ironic_lib.utils.mounted', autospec=True) -@mock.patch('ironic_lib.disk_utils.list_partitions', autospec=True) +@mock.patch('ironic_python_agent.disk_utils.list_partitions', autospec=True) @mock.patch('ironic_python_agent.hardware.dispatch_to_managers', lambda _call: '/dev/fake') class TestFindPartitionWithPath(base.IronicAgentTest): diff --git a/ironic_python_agent/tests/unit/test_partition_utils.py b/ironic_python_agent/tests/unit/test_partition_utils.py index 28fc817a1..e99ca4584 100644 --- a/ironic_python_agent/tests/unit/test_partition_utils.py +++ b/ironic_python_agent/tests/unit/test_partition_utils.py @@ -15,8 +15,6 @@ import shutil import tempfile from unittest import mock -from ironic_lib import disk_partitioner -from ironic_lib import disk_utils from ironic_lib import exception from ironic_lib import qemu_img from ironic_lib import utils @@ -24,6 +22,8 @@ from oslo_concurrency import processutils from oslo_config import cfg import requests +from ironic_python_agent import disk_partitioner +from ironic_python_agent import disk_utils from ironic_python_agent import errors from ironic_python_agent import hardware from ironic_python_agent import partition_utils @@ -168,7 +168,7 @@ class GetLabelledPartitionTestCases(base.IronicAgentTest): self.node_uuid) self.assertEqual(part_result, result) execute_calls = [ - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('lsblk', '-Po', 'name,label', self.dev, check_exit_code=[0, 1], use_standard_locale=True) @@ -184,7 +184,7 @@ class GetLabelledPartitionTestCases(base.IronicAgentTest): self.node_uuid) self.assertEqual(part_result, result) execute_calls = [ - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('lsblk', '-Po', 'name,label', self.dev, check_exit_code=[0, 1], use_standard_locale=True) @@ -199,7 +199,7 @@ class GetLabelledPartitionTestCases(base.IronicAgentTest): self.node_uuid) self.assertIsNone(result) execute_calls = [ - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('lsblk', '-Po', 'name,label', self.dev, check_exit_code=[0, 1], use_standard_locale=True) @@ -218,7 +218,7 @@ class GetLabelledPartitionTestCases(base.IronicAgentTest): self.dev, self.config_part_label, self.node_uuid) execute_calls = [ - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('lsblk', '-Po', 'name,label', self.dev, check_exit_code=[0, 1], use_standard_locale=True) @@ -234,7 +234,7 @@ class GetLabelledPartitionTestCases(base.IronicAgentTest): self.dev, self.config_part_label, self.node_uuid) execute_calls = [ - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('lsblk', '-Po', 'name,label', self.dev, check_exit_code=[0, 1], use_standard_locale=True) @@ -701,9 +701,9 @@ class CreateConfigDriveTestCases(base.IronicAgentTest): self.dev), mock.call('sync'), mock.call('udevadm', 'settle'), - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('udevadm', 'settle'), - mock.call('sgdisk', '-v', self.dev, run_as_root=True), + mock.call('sgdisk', '-v', self.dev), mock.call('udevadm', 'settle'), mock.call('test', '-e', expected_part, attempts=15, delay_on_retry=True) @@ -762,9 +762,9 @@ class CreateConfigDriveTestCases(base.IronicAgentTest): self.dev), mock.call('sync'), mock.call('udevadm', 'settle'), - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('udevadm', 'settle'), - mock.call('sgdisk', '-v', self.dev, run_as_root=True), + mock.call('sgdisk', '-v', self.dev), mock.call('udevadm', 'settle'), mock.call('test', '-e', expected_part, attempts=15, @@ -828,9 +828,9 @@ class CreateConfigDriveTestCases(base.IronicAgentTest): self.dev), mock.call('sync'), mock.call('udevadm', 'settle'), - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('udevadm', 'settle'), - mock.call('sgdisk', '-v', self.dev, run_as_root=True), + mock.call('sgdisk', '-v', self.dev), mock.call('udevadm', 'settle'), mock.call('test', '-e', expected_part, attempts=15, delay_on_retry=True) @@ -931,9 +931,9 @@ class CreateConfigDriveTestCases(base.IronicAgentTest): parted_call, mock.call('sync'), mock.call('udevadm', 'settle'), - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('udevadm', 'settle'), - mock.call('sgdisk', '-v', self.dev, run_as_root=True), + mock.call('sgdisk', '-v', self.dev), mock.call('udevadm', 'settle'), mock.call('test', '-e', expected_part, attempts=15, delay_on_retry=True) @@ -1031,9 +1031,9 @@ class CreateConfigDriveTestCases(base.IronicAgentTest): 'fat32', '-64MiB', '-0'), mock.call('sync'), mock.call('udevadm', 'settle'), - mock.call('partprobe', self.dev, run_as_root=True, attempts=10), + mock.call('partprobe', self.dev, attempts=10), mock.call('udevadm', 'settle'), - mock.call('sgdisk', '-v', self.dev, run_as_root=True), + mock.call('sgdisk', '-v', self.dev), ]) self.assertEqual(2, mock_list_partitions.call_count) @@ -1226,7 +1226,8 @@ class CreateConfigDriveTestCases(base.IronicAgentTest): # NOTE(TheJulia): trigger_device_rescan is systemwide thus pointless # to execute in the file test case. Also, CI unit test jobs lack sgdisk. @mock.patch.object(disk_utils, 'trigger_device_rescan', autospec=True) -@mock.patch.object(utils, 'wait_for_disk_to_become_available', autospec=True) +@mock.patch.object(disk_utils, 'wait_for_disk_to_become_available', + autospec=True) @mock.patch.object(disk_utils, 'is_block_device', autospec=True) @mock.patch.object(disk_utils, 'block_uuid', autospec=True) @mock.patch.object(disk_utils, 'dd', autospec=True) diff --git a/ironic_python_agent/tests/unit/test_raid_utils.py b/ironic_python_agent/tests/unit/test_raid_utils.py index f68796056..fda7795ca 100644 --- a/ironic_python_agent/tests/unit/test_raid_utils.py +++ b/ironic_python_agent/tests/unit/test_raid_utils.py @@ -12,10 +12,10 @@ from unittest import mock -from ironic_lib import disk_utils from ironic_lib import utils as ilib_utils from oslo_concurrency import processutils +from ironic_python_agent import disk_utils from ironic_python_agent import errors from ironic_python_agent import hardware from ironic_python_agent import raid_utils