Merge "Install bootloader on os disk for fake raids"

fuel_agent/drivers/nailgun.py

@@ -130,6 +130,48 @@ class Nailgun(BaseDataDriver):
             lambda x: x['type'] == 'disk' and x['size'] > 0,
             self.partition_data())
 
+    @property
+    def boot_disks(self):
+        # FIXME(agordeev): NVMe drives should be skipped as
+        # accessing such drives during the boot typically
+        # requires using UEFI which is still not supported
+        # by fuel-agent (it always installs BIOS variant of
+        # grub)
+        # * grub bug (http://savannah.gnu.org/bugs/?41883)
+        # NOTE(kozhukalov): On some hardware GRUB is not able
+        # to see disks larger than 2T due to firmware bugs,
+        # so we'd better avoid placing /boot on such
+        # huge disks if it is possible.
+        disks = self.small_ks_disks or self.ks_disks
+        suitable_disks = [
+            disk for disk in disks
+            if ('nvme' not in disk['name'] and self._is_boot_disk(disk))
+        ]
+        # FIXME(agordeev): if we have rootfs on fake raid, then /boot should
+        # land on it too. We can't proceed with grub-install otherwise.
+        md_boot_disks = [
+            disk for disk in self.md_os_disks if disk in suitable_disks]
+        if md_boot_disks:
+            return md_boot_disks
+        else:
+            return suitable_disks
+
+    def _is_boot_disk(self, disk):
+        return any(v["type"] in ('partition', 'raid') and
+                   v.get("mount") == "/boot"
+                   for v in disk["volumes"])
+
+    def _is_os_volume(self, vol):
+        return vol['size'] > 0 and vol['type'] == 'pv' and vol['vg'] == 'os'
+
+    def _is_os_disk(self, disk):
+        return any(self._is_os_volume(vol) for vol in disk['volumes'])
+
+    @property
+    def md_os_disks(self):
+        return [d for d in self.ks_disks
+                if d['name'].startswith('md') and self._is_os_disk(d)]
+
     @property
     def small_ks_disks(self):
         """Get those disks which are smaller than 2T"""
@@ -256,17 +298,19 @@ class Nailgun(BaseDataDriver):
             LOG.debug('Adding gpt table on disk %s' % disk['name'])
             parted = partition_scheme.add_parted(
                 name=self._disk_dev(disk), label='gpt')
-            # we install bootloader on every disk
+            if disk in self.boot_disks:
-            LOG.debug('Adding bootloader stage0 on disk %s' % disk['name'])
+                # we install bootloader only on every suitable disk
-            parted.install_bootloader = True
+                LOG.debug('Adding bootloader stage0 on disk %s' % disk['name'])
-            # legacy boot partition
+                parted.install_bootloader = True
-            LOG.debug('Adding bios_grub partition on disk %s: size=24' %
+
-                      disk['name'])
+                # legacy boot partition
-            parted.add_partition(size=24, flags=['bios_grub'])
+                LOG.debug('Adding bios_grub partition on disk %s: size=24' %
-            # uefi partition (for future use)
+                          disk['name'])
-            LOG.debug('Adding UEFI partition on disk %s: size=200' %
+                parted.add_partition(size=24, flags=['bios_grub'])
-                      disk['name'])
+                # uefi partition (for future use)
-            parted.add_partition(size=200)
+                LOG.debug('Adding UEFI partition on disk %s: size=200' %
+                          disk['name'])
+                parted.add_partition(size=200)
 
             LOG.debug('Looping over all volumes on disk %s' % disk['name'])
             for volume in disk['volumes']:
@@ -327,19 +371,7 @@ class Nailgun(BaseDataDriver):
 
                     elif volume.get('mount') == '/boot' \
                             and not self._boot_partition_done \
-                            and 'nvme' not in disk['name'] \
+                            and disk in self.boot_disks:
-                            and (disk in self.small_ks_disks or
-                                 not self.small_ks_disks):
-                        # FIXME(agordeev): NVMe drives should be skipped as
-                        # accessing such drives during the boot typically
-                        # requires using UEFI which is still not supported
-                        # by fuel-agent (it always installs BIOS variant of
-                        # grub)
-                        # * grub bug (http://savannah.gnu.org/bugs/?41883)
-                        # NOTE(kozhukalov): On some hardware GRUB is not able
-                        # to see disks larger than 2T due to firmware bugs,
-                        # so we'd better avoid placing /boot on such
-                        # huge disks if it is possible.
                         LOG.debug('Adding /boot partition on disk %s: '
                                   'size=%s', disk['name'], volume['size'])
                         prt = parted.add_partition(

fuel_agent/tests/test_nailgun.py

@@ -404,6 +404,25 @@ LIST_BLOCK_DEVICES_SAMPLE = [
          'ss': '512', 'ioopt': '0', 'alignoff': '0', 'pbsz': '4096',
          'ra': '256', 'ro': '0', 'maxsect': '1024'},
      'size': 500107862016},
+    {'uspec':
+        {'DEVLINKS': [
+            '/dev/disk/by-id/by-id/md-fake-raid-uuid',
+            ],
+         'ID_SERIAL_SHORT': 'fake_serial_raid',
+         'ID_WWN': 'fake_wwn_raid',
+         'DEVPATH': '/devices/virtual/block/md123',
+         'ID_MODEL': 'fake_raid',
+         'DEVNAME': '/dev/md123',
+         'MAJOR': '9',
+         'DEVTYPE': 'disk', 'MINOR': '123'},
+     'startsec': '0',
+     'device': '/dev/md123',
+     'espec': {'state': 'running', 'timeout': '30', 'removable': '0'},
+     'bspec': {
+         'sz': '976773168', 'iomin': '4096', 'size64': '500107862016',
+         'ss': '512', 'ioopt': '0', 'alignoff': '0', 'pbsz': '4096',
+         'ra': '256', 'ro': '0', 'maxsect': '1024'},
+     'size': 500107862016},
 ]
 
 LIST_BLOCK_DEVICES_SAMPLE_NVME = [
@@ -784,6 +803,116 @@ SINGLE_NVME_DISK_KS_SPACES = [
 ]
 
 
+FAKE_RAID_DISK_KS_SPACES = [
+    {
+        "name": "sda",
+        "extra": ["sda"],
+        "free_space": 1024,
+        "volumes": [
+            {
+                "type": "boot",
+                "size": 300
+            },
+            {
+                "mount": "/boot",
+                "size": 200,
+                "type": "raid",
+                "file_system": "ext2",
+                "name": "Boot"
+            },
+            {
+                "mount": "/var",
+                "size": 200,
+                "type": "partition",
+                "file_system": "ext4",
+                "name": "Var"
+            },
+        ],
+        "type": "disk",
+        "id": "sda",
+        "size": 2097153
+    },
+    {
+        "name": "sdb",
+        "extra": ["sdb"],
+        "free_space": 1024,
+        "volumes": [
+            {
+                "type": "boot",
+                "size": 300
+            },
+            {
+                "mount": "/boot",
+                "size": 200,
+                "type": "raid",
+                "file_system": "ext2",
+                "name": "Boot"
+            },
+            {
+                "mount": "/tmp",
+                "size": 200,
+                "type": "partition",
+                "file_system": "ext2",
+                "name": "TMP"
+            },
+        ],
+        "type": "disk",
+        "id": "sdb",
+        "size": 2097153
+    },
+    {
+        "name": "md123",
+        "extra": ["md123"],
+        "free_space": 1024,
+        "volumes": [
+            {
+                "type": "boot",
+                "size": 300
+            },
+            {
+                "mount": "/boot",
+                "size": 200,
+                "type": "raid",
+                "file_system": "ext2",
+                "name": "Boot"
+            },
+            {
+                "lvm_meta_size": 64,
+                "size": 271370,
+                "type": "pv",
+                "vg": "os"
+            },
+
+        ],
+        "type": "disk",
+        "id": "md123",
+        "size": 2097153
+    },
+    {
+        "id": "os",
+        "label": "Base System",
+        "min_size": 55296,
+        "type": "vg",
+        "volumes": [
+            {
+                "file_system": "ext4",
+                "mount": "/",
+                "name": "root",
+                "size": 267210,
+                "type": "lv"
+            },
+            {
+                "file_system": "swap",
+                "mount": "swap",
+                "name": "swap",
+                "size": 4096,
+                "type": "lv"
+            }
+        ],
+    },
+]
+
+
 class TestNailgunMatch(unittest2.TestCase):
     def test_match_device_by_id_matches(self):
         # matches by 'by-id' links
@@ -897,6 +1026,72 @@ class TestNailgunMatch(unittest2.TestCase):
         self.assertFalse(nailgun.match_device(fake_hu_disk, fake_ks_disk))
 
 
+class TestNailgunBootDisks(unittest2.TestCase):
+    class PropertyMock(mock.Mock):
+        def __get__(self, instance, owner):
+            return self()
+
+    nvme_disk = {
+        'name': 'nvmen1', 'size': 5,
+        'volumes': [{'type': 'raid', 'mount': '/boot', 'size': 1}],
+    }
+    disks = [
+        {'name': 'sda', 'size': 5,
+         'volumes': [{'type': 'partition', 'mount': '/boot',
+                     'size': 1}],
+         },
+        {'name': 'sdb', 'size': 5,
+         'volumes': [{'type': 'raid', 'mount': '/boot', 'size': 1}],
+         },
+    ]
+    big_disk = {
+        'name': '2big', 'size': 555555555,
+        'volumes': [{'type': 'raid', 'mount': '/boot', 'size': 1}],
+    }
+    fake_raid = {
+        'name': 'md123', 'size': 5,
+        'volumes': [{'type': 'raid', 'mount': '/boot', 'size': 1},
+                    {'type': 'pv', 'vg': 'os', 'size': 1}],
+    }
+    non_os_fake_raid = {
+        'name': 'md456', 'size': 5,
+        'volumes': [{'type': 'raid', 'mount': '/boot', 'size': 1},
+                    {'type': 'pv', 'vg': 'image', 'size': 1}],
+    }
+
+    def _check_boot_disks(self, ks_disks_return_value,
+                          not_expected_disk, expected_disks):
+        with mock.patch.object(nailgun.Nailgun, '__init__', return_value=None):
+            ks_disks = self.PropertyMock()
+            with mock.patch.object(nailgun.Nailgun, 'ks_disks', ks_disks):
+                drv = nailgun.Nailgun('fake_data')
+                ks_disks.return_value = ks_disks_return_value
+                self.assertNotIn(not_expected_disk, drv.boot_disks)
+                self.assertEqual(expected_disks, drv.boot_disks)
+
+    def test_md_boot_disk(self):
+        ks_disks_return_value = self.disks + [self.non_os_fake_raid] +\
+            [self.fake_raid]
+        not_expected_disk = self.non_os_fake_raid
+        expected_disks = [self.fake_raid]
+        self._check_boot_disks(ks_disks_return_value, not_expected_disk,
+                               expected_disks)
+
+    def test_only_small_boot_disks(self):
+        ks_disks_return_value = self.disks + [self.big_disk]
+        not_expected_disk = self.big_disk
+        expected_disks = self.disks
+        self._check_boot_disks(ks_disks_return_value, not_expected_disk,
+                               expected_disks)
+
+    def test_boot_disks_no_nvme(self):
+        ks_disks_return_value = self.disks + [self.nvme_disk]
+        not_expected_disk = self.nvme_disk
+        expected_disks = self.disks
+        self._check_boot_disks(ks_disks_return_value, not_expected_disk,
+                               expected_disks)
+
+
 @mock.patch.object(nailgun.Nailgun, '__init__', return_value=None)
 class TestNailgunGetOSMethods(unittest2.TestCase):
     def test_parse_operating_system_test_profiles(self, mock_nailgun):
@@ -1224,6 +1419,16 @@ class TestNailgunMockedMeta(unittest2.TestCase):
             drv.partition_scheme.fs_by_mount('/boot').device,
             '/dev/sda3')
 
+    def test_boot_partition_and_rootfs_on_fake_raid(self, mock_lbd,
+                                                    mock_image_meta):
+        data = copy.deepcopy(PROVISION_SAMPLE_DATA)
+        data['ks_meta']['pm_data']['ks_spaces'] = FAKE_RAID_DISK_KS_SPACES
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        drv = nailgun.Nailgun(data)
+        self.assertEqual(
+            drv.partition_scheme.fs_by_mount('/boot').device,
+            '/dev/md123p3')
+
     def test_boot_partition_no_boot(self, mock_lbd, mock_image_meta):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['ks_meta']['pm_data']['ks_spaces'] = NO_BOOT_KS_SPACES