From 55b479b54f8cd064144ba8d1e2e5be33b6a975c8 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 15 Jan 2018 14:44:10 +1100 Subject: [PATCH] GPT partitioning support This adds support for a GPT label type to the partitioning code. This is relatively straight-forward translation of the partition config into a sgparted command-line and subsequent call. A unit test is added based on a working GPT/EFI configuration and the fedora-minimal functional test is updated to build a single-partition GPT based using the new block-device-gpt override element. See notes in the sample configuration files about partition requirements and types. Documentation has been updated. Co-Authored-By: Marcin Juszkiewicz Change-Id: I6b819a8071389e7e4eb4874ff7750bd192695ff2 --- .../block_device/level1/partition.py | 11 +- .../block_device/level1/partitioning.py | 114 +++++++++++++----- .../block_device/tests/config/gpt_efi.yaml | 32 +++++ .../block_device/tests/test_gpt.py | 85 +++++++++++++ .../elements/block-device-gpt/README.rst | 11 ++ .../block-device-default.yaml | 22 ++++ .../test-elements/build-succeeds/element-deps | 4 +- doc/source/user_guide/building_an_image.rst | 60 ++++++--- .../bootloader-gpt-d1047f81f3a0631b.yaml | 7 ++ tests/install_test_deps.sh | 8 ++ 10 files changed, 308 insertions(+), 46 deletions(-) create mode 100644 diskimage_builder/block_device/tests/config/gpt_efi.yaml create mode 100644 diskimage_builder/block_device/tests/test_gpt.py create mode 100644 diskimage_builder/elements/block-device-gpt/README.rst create mode 100644 diskimage_builder/elements/block-device-gpt/block-device-default.yaml create mode 100644 releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml diff --git a/diskimage_builder/block_device/level1/partition.py b/diskimage_builder/block_device/level1/partition.py index e91e1bf98..183778607 100644 --- a/diskimage_builder/block_device/level1/partition.py +++ b/diskimage_builder/block_device/level1/partition.py @@ -33,6 +33,12 @@ class PartitionNode(NodeBase): self.partitioning = parent self.prev_partition = prev_partition + # filter out some MBR only options for clarity + if self.partitioning.label == 'gpt': + if 'flags' in config and 'primary' in config['flags']: + raise BlockDeviceSetupException( + "Primary flag not supported for GPT partitions") + self.flags = set() if 'flags' in config: for f in config['flags']: @@ -47,7 +53,10 @@ class PartitionNode(NodeBase): raise BlockDeviceSetupException("No size in partition" % self.name) self.size = config['size'] - self.ptype = int(config['type'], 16) if 'type' in config else 0x83 + if self.partitioning.label == 'gpt': + self.ptype = str(config['type']) if 'type' in config else '8300' + elif self.partitioning.label == 'mbr': + self.ptype = int(config['type'], 16) if 'type' in config else 83 def get_flags(self): return self.flags diff --git a/diskimage_builder/block_device/level1/partitioning.py b/diskimage_builder/block_device/level1/partitioning.py index e8a559867..bc2a728ef 100644 --- a/diskimage_builder/block_device/level1/partitioning.py +++ b/diskimage_builder/block_device/level1/partitioning.py @@ -58,8 +58,8 @@ class Partitioning(PluginBase): raise BlockDeviceSetupException( "Partitioning config needs 'label'") self.label = config['label'] - if self.label not in ("mbr", ): - raise BlockDeviceSetupException("Label must be 'mbr'") + if self.label not in ("mbr", "gpt"): + raise BlockDeviceSetupException("Label must be 'mbr' or 'gpt'") # It is VERY important to get the alignment correct. If this # is not correct, the disk performance might be very poor. @@ -93,29 +93,9 @@ class Partitioning(PluginBase): fd.seek(0, 2) return fd.tell() - # not this is NOT a node and this is not called directly! The - # create() calls in the partition nodes this plugin has - # created are calling back into this. - def create(self): - # This is a bit of a hack. Each of the partitions is actually - # in the graph, so for every partition we get a create() call - # as the walk happens. But we only need to create the - # partition table once... - if self.already_created: - logger.info("Not creating the partitions a second time.") - return - self.already_created = True - - # the raw file on disk - image_path = self.state['blockdev'][self.base]['image'] - # the /dev/loopX device of the parent - device_path = self.state['blockdev'][self.base]['device'] - logger.info("Creating partition on [%s] [%s]", self.base, image_path) - - assert self.label == 'mbr' - - disk_size = self._size_of_block_dev(image_path) - with MBR(image_path, disk_size, self.align) as part_impl: + def _create_mbr(self): + """Create partitions with MBR""" + with MBR(self.image_path, self.disk_size, self.align) as part_impl: for part_cfg in self.partitions: part_name = part_cfg.get_name() part_bootflag = PartitionNode.flag_boot \ @@ -137,24 +117,100 @@ class Partitioning(PluginBase): # We're going to mount all partitions with kpartx # below once we're done. So the device this partition # will be seen at becomes "/dev/mapper/loop0pX" - assert device_path[:5] == "/dev/" + assert self.device_path[:5] == "/dev/" partition_device_name = "/dev/mapper/%sp%d" % \ - (device_path[5:], part_no) + (self.device_path[5:], part_no) self.state['blockdev'][part_name] \ = {'device': partition_device_name} + def _create_gpt(self): + """Create partitions with GPT""" + + cmd = ['sgdisk', self.image_path] + + # This padding gives us a little room for rounding so we don't + # go over the end of the disk + disk_free = self.disk_size - (2048 * 1024) + pnum = 1 + + for p in self.partitions: + args = {} + args['pnum'] = pnum + args['name'] = '"%s"' % p.get_name() + args['type'] = '%s' % p.get_type() + + # convert from a relative/string size to bytes + size = parse_rel_size_spec(p.get_size(), disk_free)[1] + + # We keep track in bytes, but specify things to sgdisk in + # megabytes so it can align on sensible boundaries. And + # create partitions right after previous so no need to + # calculate start/end - just size. + assert size <= disk_free + args['size'] = size // (1024 * 1024) + + new_cmd = ("-n {pnum}:0:+{size}M -t {pnum}:{type} " + "-c {pnum}:{name}".format(**args)) + cmd.extend(new_cmd.strip().split(' ')) + + # Fill the state; we mount all partitions with kpartx + # below once we're done. So the device this partition + # will be seen at becomes "/dev/mapper/loop0pX" + assert self.device_path[:5] == "/dev/" + device_name = "/dev/mapper/%sp%d" % (self.device_path[5:], pnum) + self.state['blockdev'][p.get_name()] \ + = {'device': device_name} + + disk_free = disk_free - size + pnum = pnum + 1 + logger.debug("Partition %s added, %s remaining in disk", + pnum, disk_free) + + logger.debug("cmd: %s", ' '.join(cmd)) + exec_sudo(cmd) + + # not this is NOT a node and this is not called directly! The + # create() calls in the partition nodes this plugin has + # created are calling back into this. + def create(self): + # This is a bit of a hack. Each of the partitions is actually + # in the graph, so for every partition we get a create() call + # as the walk happens. But we only need to create the + # partition table once... + if self.already_created: + logger.info("Not creating the partitions a second time.") + return + self.already_created = True + + # the raw file on disk + self.image_path = self.state['blockdev'][self.base]['image'] + # the /dev/loopX device of the parent + self.device_path = self.state['blockdev'][self.base]['device'] + # underlying size + self.disk_size = self._size_of_block_dev(self.image_path) + + logger.info("Creating partition on [%s] [%s]", + self.base, self.image_path) + + assert self.label in ('mbr', 'gpt') + + if self.label == 'mbr': + self._create_mbr() + elif self.label == 'gpt': + self._create_gpt() + # "saftey sync" to make sure the partitions are written exec_sudo(["sync"]) # now all the partitions are created, get device-mapper to # mount them if not os.path.exists("/.dockerenv"): - exec_sudo(["kpartx", "-avs", device_path]) + exec_sudo(["kpartx", "-avs", self.device_path]) else: # If running inside Docker, make our nodes manually, # because udev will not be working. kpartx cannot run in # sync mode in docker. - exec_sudo(["kpartx", "-av", device_path]) + exec_sudo(["kpartx", "-av", self.device_path]) exec_sudo(["dmsetup", "--noudevsync", "mknodes"]) return diff --git a/diskimage_builder/block_device/tests/config/gpt_efi.yaml b/diskimage_builder/block_device/tests/config/gpt_efi.yaml new file mode 100644 index 000000000..9ef2f682f --- /dev/null +++ b/diskimage_builder/block_device/tests/config/gpt_efi.yaml @@ -0,0 +1,32 @@ +# A sample config that has GPT/bios and EFI boot partitions + +- local_loop: + name: image0 + +- partitioning: + base: image0 + label: gpt + partitions: + - name: ESP + type: 'EF00' + size: 8MiB + mkfs: + type: vfat + mount: + mount_point: /boot/efi + fstab: + options: "defaults" + fsck-passno: 1 + - name: BSP + type: 'EF02' + size: 8MiB + - name: root + type: '8300' + size: 100% + mkfs: + type: ext4 + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 diff --git a/diskimage_builder/block_device/tests/test_gpt.py b/diskimage_builder/block_device/tests/test_gpt.py new file mode 100644 index 000000000..bc2009800 --- /dev/null +++ b/diskimage_builder/block_device/tests/test_gpt.py @@ -0,0 +1,85 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import logging +import mock +import os + +import diskimage_builder.block_device.tests.test_config as tc + +from diskimage_builder.block_device.blockdevice import BlockDeviceState +from diskimage_builder.block_device.config import config_tree_to_graph +from diskimage_builder.block_device.config import create_graph +from diskimage_builder.block_device.level0.localloop import image_create +from diskimage_builder.block_device.level1.partition import PartitionNode + +logger = logging.getLogger(__name__) + + +class TestGPT(tc.TestGraphGeneration): + + @mock.patch('diskimage_builder.block_device.level1.partitioning.exec_sudo') + def test_gpt_efi(self, mock_exec_sudo): + # Test the command-sequence for a GPT/EFI partition setup + tree = self.load_config_file('gpt_efi.yaml') + config = config_tree_to_graph(tree) + + state = BlockDeviceState() + + graph, call_order = create_graph(config, self.fake_default_config, + state) + + # Create a fake temp backing file (we check the size of it, + # etc). + # TODO(ianw): exec_sudo is generically mocked out, thus the + # actual creation is mocked out ... but we could do this + # without root and use parted to create the partitions on this + # for slightly better testing. An exercise for another day... + self.tmp_dir = fixtures.TempDir() + self.useFixture(self.tmp_dir) + self.image_path = os.path.join(self.tmp_dir.path, "image.raw") + # should be sparse... + image_create(self.image_path, 1024 * 1024 * 1024) + logger.debug("Temp image in %s", self.image_path) + + # Fake state for the loopback device + state['blockdev'] = {} + state['blockdev']['image0'] = {} + state['blockdev']['image0']['image'] = self.image_path + state['blockdev']['image0']['device'] = "/dev/loopX" + + for node in call_order: + if isinstance(node, PartitionNode): + node.create() + + # check the parted call looks right + parted_cmd = ('sgdisk %s ' + '-n 1:0:+8M -t 1:EF00 -c 1:"ESP" ' + '-n 2:0:+8M -t 2:EF02 -c 2:"BSP" ' + '-n 3:0:+1006M -t 3:8300 -c 3:"root"' + % self.image_path) + cmd_sequence = [ + mock.call(parted_cmd.split(' ')), + mock.call(['sync']), + mock.call(['kpartx', '-avs', '/dev/loopX']) + ] + self.assertEqual(mock_exec_sudo.call_count, len(cmd_sequence)) + mock_exec_sudo.assert_has_calls(cmd_sequence) + + # Check two new partitions appear in state correctly + self.assertDictEqual(state['blockdev']['ESP'], + {'device': '/dev/mapper/loopXp1'}) + self.assertDictEqual(state['blockdev']['BSP'], + {'device': '/dev/mapper/loopXp2'}) + self.assertDictEqual(state['blockdev']['root'], + {'device': '/dev/mapper/loopXp3'}) diff --git a/diskimage_builder/elements/block-device-gpt/README.rst b/diskimage_builder/elements/block-device-gpt/README.rst new file mode 100644 index 000000000..64d0ffbd8 --- /dev/null +++ b/diskimage_builder/elements/block-device-gpt/README.rst @@ -0,0 +1,11 @@ +================ +Block Device GPT +================ + +This is an override for the default block-device configuration +provided in the ``vm`` element to get a GPT based single-partition +disk, rather than the default MBR. + +Note this provides the extra `BIOS boot partition +`__ as required for +non-EFI boot environments. diff --git a/diskimage_builder/elements/block-device-gpt/block-device-default.yaml b/diskimage_builder/elements/block-device-gpt/block-device-default.yaml new file mode 100644 index 000000000..01223dda2 --- /dev/null +++ b/diskimage_builder/elements/block-device-gpt/block-device-default.yaml @@ -0,0 +1,22 @@ +# Default single partition loopback using a GPT based partition table + +- local_loop: + name: image0 + +- partitioning: + base: image0 + label: gpt + partitions: + - name: BSP + type: 'EF02' + size: 8MiB + - name: root + flags: [ boot ] + size: 100% + mkfs: + type: ext4 + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 diff --git a/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps b/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps index 7791c84fc..20b98fe31 100644 --- a/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps +++ b/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps @@ -1 +1,3 @@ -openstack-ci-mirrors \ No newline at end of file +block-device-gpt +openstack-ci-mirrors +vm diff --git a/doc/source/user_guide/building_an_image.rst b/doc/source/user_guide/building_an_image.rst index a9edf5317..720240cd4 100644 --- a/doc/source/user_guide/building_an_image.rst +++ b/doc/source/user_guide/building_an_image.rst @@ -75,7 +75,8 @@ There are currently two defaults: The user can overwrite the default handling by setting the environment variable `DIB_BLOCK_DEVICE_CONFIG`. This variable must hold YAML -structured configuration data. +structured configuration data or be a ``file://`` URL reference to a +on-disk configuration file. The default when using the `vm` element is: @@ -247,8 +248,8 @@ encrypted, ...) and create partition information in it. The symbolic name for this module is `partitioning`. -Currently the only supported partitioning layout is Master Boot Record -`MBR`. +MBR +*** It is possible to create primary or logical partitions or a mix of them. The numbering of the primary partitions will start at 1, @@ -267,19 +268,27 @@ partitions. Partitions are created in the order they are configured. Primary partitions - if needed - must be first in the list. +GPT +*** + +GPT partitioning requires the ``sgdisk`` tool to be available. + +Options +******* + There are the following key / value pairs to define one partition table: base - (mandatory) The base device where to create the partitions in. + (mandatory) The base device to create the partitions in. label - (mandatory) Possible values: 'mbr' - This uses the Master Boot Record (MBR) layout for the disk. - (There are currently plans to add GPT later on.) + (mandatory) Possible values: 'mbr', 'gpt' + Configure use of either the Master Boot Record (MBR) or GUID + Partition Table (GPT) formats align - (optional - default value '1MiB') + (optional - default value '1MiB'; MBR only) Set the alignment of the partition. This must be a multiple of the block size (i.e. 512 bytes). The default of 1MiB (~ 2048 * 512 bytes blocks) is the default for modern systems and known to @@ -308,9 +317,9 @@ flags (optional) List of flags for the partition. Default: empty. Possible values: - boot + boot (MBR only) Sets the boot flag for the partition - primary + primary (MBR only) Partition should be a primary partition. If not set a logical partition will be created. @@ -321,10 +330,15 @@ size based on the remaining free space. type (optional) - The partition type stored in the MBR partition table entry. The - default value is '0x83' (Linux Default partition). Any valid one + The partition type stored in the MBR or GPT partition table entry. + + For MBR the default value is '0x83' (Linux Default partition). Any valid one byte hexadecimal value may be specified here. + For GPT the default value is '8300' (Linux Default partition). Any valid two + byte hexadecimal value may be specified here. Due to ``sgdisk`` leading '0x' + should not be used. + Example: .. code-block:: yaml @@ -350,12 +364,28 @@ Example: - name: data2 size: 100% + - partitioning: + base: gpt_image + label: gpt + partitions: + - name: ESP + type: EF00 + size: 16MiB + - name: data1 + size: 1GiB + - name: lvmdata + type: 8E00 + size: 100% + On the `image0` two partitions are created. The size of the first is 1GiB, the second uses the remaining free space. On the `data_image` -three partitions are created: all are about 1/3 of the disk size. +three partitions are created: all are about 1/3 of the disk size. On +the `gpt_image` three partitions are created: 16MiB one for EFI +bootloader, 1GiB Linux filesystem one and rest of disk will be used +for LVM partition. -Module: Lvm -··········· +Module: LVM +........... This module generates volumes on existing block devices. This means that it is possible to take any previous created partition, and create volumes information diff --git a/releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml b/releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml new file mode 100644 index 000000000..6941056be --- /dev/null +++ b/releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + GPT support is added to the bootloader; see documentation for + configuration examples. This should be considered a technology + preview; there may be minor behaviour modifications as we enable + UEFI and support across more architectures. \ No newline at end of file diff --git a/tests/install_test_deps.sh b/tests/install_test_deps.sh index 17175417a..befea9a01 100755 --- a/tests/install_test_deps.sh +++ b/tests/install_test_deps.sh @@ -9,6 +9,8 @@ sudo apt-get install -y --force-yes \ bzip2 \ debootstrap \ docker.io \ + dosfstools \ + gdisk \ inetutils-ping \ lsb-release \ kpartx \ @@ -22,6 +24,8 @@ sudo apt-get install -y --force-yes \ dpkg \ debootstrap \ docker \ + dosfstools \ + gdisk \ kpartx \ util-linux \ qemu-img \ @@ -30,6 +34,8 @@ sudo apt-get install -y --force-yes \ bzip2 \ debootstrap \ docker \ + dosfstools \ + gdisk \ kpartx \ util-linux \ python-pyliblzma \ @@ -40,6 +46,8 @@ sudo apt-get install -y --force-yes \ app-emulation/qemu \ dev-python/pyyaml \ sys-block/parted \ + sys-apps/gptfdisk \ sys-fs/multipath-tools \ + sys-fs/dosfstools \ qemu-img \ yum-utils