KVM and XEN Disk Management Parity

Implements blueprint disk-configuration-parity

This change splits local_gb into root_gb and ephemeral_gb. libvirt
interpreted local_gb as what ephemeral_gb is now, whereas XenAPI
interpreted local_gb as what root_gb is now.

Change-Id: I496600991bac1e990326d4ded1607fee08209d68

nova/flags.py

@@ -261,7 +261,7 @@ DEFINE_string('my_ip', _get_my_ip(), 'host ip address')
 DEFINE_list('region_list',
             [],
             'list of region=fqdn pairs separated by commas')
-DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake')
+DEFINE_string('connection_type', None, 'libvirt, xenapi or fake')
 DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID')
 DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key')
 # NOTE(sirp): my_ip interpolation doesn't work within nested structures
@@ -428,6 +428,10 @@ DEFINE_bool('start_guests_on_host_boot', False,
             'Whether to restart guests when the host reboots')
 DEFINE_bool('resume_guests_state_on_host_boot', False,
             'Whether to start guests, that was running before the host reboot')
+DEFINE_string('default_ephemeral_format',
+              None,
+              'The default format a ephemeral_volume will be formatted '
+              'with on creation.')
 
 DEFINE_string('root_helper', 'sudo',
               'Command prefix to use for running commands as root')

nova/tests/scheduler/test_distributed_scheduler.py

@@ -104,7 +104,8 @@ class DistributedSchedulerTestCase(test.TestCase):
         self.stubs.Set(db, 'zone_get_all', fake_zone_get_all)
 
         fake_context = context.RequestContext('user', 'project')
-        request_spec = {'instance_type': {'memory_mb': 1, 'local_gb': 1},
+        request_spec = {'instance_type': {'memory_mb': 1, 'root_gb': 1,
+                                          'ephemeral_gb': 0},
                         'instance_properties': {'project_id': 1}}
         self.assertRaises(exception.NoValidHost, sched.schedule_run_instance,
                           fake_context, request_spec)
@@ -219,7 +220,8 @@ class DistributedSchedulerTestCase(test.TestCase):
         self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method)
 
         request_spec = {'num_instances': 10,
-                        'instance_type': {'memory_mb': 512, 'local_gb': 512},
+                        'instance_type': {'memory_mb': 512, 'root_gb': 512,
+                                          'ephemeral_gb': 0},
                         'instance_properties': {'project_id': 1}}
         self.mox.ReplayAll()
         weighted_hosts = sched._schedule(fake_context, 'compute',
@@ -260,10 +262,12 @@ class DistributedSchedulerTestCase(test.TestCase):
         self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method)
 
         request_spec = {'num_instances': 10,
-                        'instance_type': {'memory_mb': 512, 'local_gb': 512},
+                        'instance_type': {'memory_mb': 512, 'root_gb': 512,
+                                          'ephemeral_gb': 256},
                         'instance_properties': {'project_id': 1,
                                                 'memory_mb': 512,
-                                                'local_gb': 512,
+                                                'root_gb': 512,
+                                                'ephemeral_gb': 0,
                                                 'vcpus': 1}}
         filter_properties = {'local_zone_only': True}
         self.mox.ReplayAll()

nova/tests/test_compute.py

@@ -151,6 +151,8 @@ class BaseTestCase(test.TestCase):
         type_id = instance_types.get_instance_type_by_name(type_name)['id']
         inst['instance_type_id'] = type_id
         inst['ami_launch_index'] = 0
+        inst['root_gb'] = 0
+        inst['ephemeral_gb'] = 0
         inst.update(params)
         return db.instance_create(self.context, inst)
 
@@ -168,7 +170,8 @@ class BaseTestCase(test.TestCase):
         inst['name'] = 'm1.small'
         inst['memory_mb'] = '1024'
         inst['vcpus'] = '1'
-        inst['local_gb'] = '20'
+        inst['root_gb'] = '20'
+        inst['ephemeral_gb'] = '10'
         inst['flavorid'] = '1'
         inst['swap'] = '2048'
         inst['rxtx_factor'] = 1
@@ -1458,7 +1461,7 @@ class ComputeAPITestCase(BaseTestCase):
         """Test an instance type with too little disk space"""
 
         inst_type = instance_types.get_default_instance_type()
-        inst_type['local_gb'] = 1
+        inst_type['root_gb'] = 1
 
         def fake_show(*args):
             img = copy(self.fake_image)
@@ -1470,7 +1473,7 @@ class ComputeAPITestCase(BaseTestCase):
             self.compute_api.create, self.context, inst_type, None)
 
         # Now increase the inst_type disk space and make sure all is fine.
-        inst_type['local_gb'] = 2
+        inst_type['root_gb'] = 2
         (refs, resv_id) = self.compute_api.create(self.context,
                 inst_type, None)
         db.instance_destroy(self.context, refs[0]['id'])
@@ -1479,7 +1482,7 @@ class ComputeAPITestCase(BaseTestCase):
         """Test an instance type with just enough ram and disk space"""
 
         inst_type = instance_types.get_default_instance_type()
-        inst_type['local_gb'] = 2
+        inst_type['root_gb'] = 2
         inst_type['memory_mb'] = 2
 
         def fake_show(*args):
@@ -1497,7 +1500,7 @@ class ComputeAPITestCase(BaseTestCase):
         """Test an instance type with no min_ram or min_disk"""
 
         inst_type = instance_types.get_default_instance_type()
-        inst_type['local_gb'] = 1
+        inst_type['root_gb'] = 1
         inst_type['memory_mb'] = 1
 
         def fake_show(*args):
@@ -1951,7 +1954,7 @@ class ComputeAPITestCase(BaseTestCase):
         self.stubs.Set(fake_image._FakeImageService, 'show', fake_show)
 
         instance = self._create_fake_instance()
-        inst_params = {'local_gb': 2, 'memory_mb': 256}
+        inst_params = {'root_gb': 2, 'memory_mb': 256}
         instance['instance_type'].update(inst_params)
 
         image = self.compute_api.snapshot(self.context, instance, 'snap1',
@@ -2777,12 +2780,12 @@ class ComputeAPITestCase(BaseTestCase):
         self.compute.terminate_instance(self.context, instance['uuid'])
 
     def test_volume_size(self):
-        local_size = 2
+        ephemeral_size = 2
         swap_size = 3
-        inst_type = {'local_gb': local_size, 'swap': swap_size}
+        inst_type = {'ephemeral_gb': ephemeral_size, 'swap': swap_size}
         self.assertEqual(self.compute_api._volume_size(inst_type,
                                                        'ephemeral0'),
-                         local_size)
+                         ephemeral_size)
         self.assertEqual(self.compute_api._volume_size(inst_type,
                                                        'ephemeral1'),
                          0)

nova/tests/test_compute_utils.py

@@ -66,6 +66,8 @@ class UsageInfoTestCase(test.TestCase):
         type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
         inst['instance_type_id'] = type_id
         inst['ami_launch_index'] = 0
+        inst['root_gb'] = 0
+        inst['ephemeral_gb'] = 0
         inst.update(params)
         return db.instance_create(self.context, inst)['id']
 

nova/tests/test_instance_types.py

@@ -68,13 +68,14 @@ class InstanceTypeTestCase(test.TestCase):
         original_list = instance_types.get_all_types()
 
         # create new type and make sure values stick
-        inst_type = instance_types.create(name, 256, 1, 120, flavorid)
+        inst_type = instance_types.create(name, 256, 1, 120, 100, flavorid)
         inst_type_id = inst_type['id']
         self.assertEqual(inst_type['flavorid'], flavorid)
         self.assertEqual(inst_type['name'], name)
         self.assertEqual(inst_type['memory_mb'], 256)
         self.assertEqual(inst_type['vcpus'], 1)
-        self.assertEqual(inst_type['local_gb'], 120)
+        self.assertEqual(inst_type['root_gb'], 120)
+        self.assertEqual(inst_type['ephemeral_gb'], 100)
         self.assertEqual(inst_type['swap'], 0)
         self.assertEqual(inst_type['rxtx_factor'], 1)
 
@@ -108,22 +109,23 @@ class InstanceTypeTestCase(test.TestCase):
     def test_invalid_create_args_should_fail(self):
         """Ensures that instance type creation fails with invalid args"""
         invalid_sigs = [
-            (('Zero memory', 0, 1, 10, 'flavor1'), {}),
+            (('Zero memory', 0, 1, 10, 20, 'flavor1'), {}),
-            (('Negative memory', -256, 1, 10, 'flavor1'), {}),
+            (('Negative memory', -256, 1, 10, 20, 'flavor1'), {}),
-            (('Non-integer memory', 'asdf', 1, 10, 'flavor1'), {}),
+            (('Non-integer memory', 'asdf', 1, 10, 20, 'flavor1'), {}),
 
-            (('Zero vcpus', 256, 0, 10, 'flavor1'), {}),
+            (('Zero vcpus', 256, 0, 10, 20, 'flavor1'), {}),
-            (('Negative vcpus', 256, -1, 10, 'flavor1'), {}),
+            (('Negative vcpus', 256, -1, 10, 20, 'flavor1'), {}),
-            (('Non-integer vcpus', 256, 'a', 10, 'flavor1'), {}),
+            (('Non-integer vcpus', 256, 'a', 10, 20, 'flavor1'), {}),
 
-            (('Negative storage', 256, 1, -1, 'flavor1'), {}),
+            (('Negative storage', 256, 1, -1, 20, 'flavor1'), {}),
-            (('Non-integer storage', 256, 1, 'a', 'flavor1'), {}),
+            (('Non-integer storage', 256, 1, 'a', 20, 'flavor1'), {}),
 
-            (('Negative swap', 256, 1, 10, 'flavor1'), {'swap': -1}),
+            (('Negative swap', 256, 1, 10, 20, 'flavor1'), {'swap': -1}),
-            (('Non-integer swap', 256, 1, 10, 'flavor1'), {'swap': -1}),
+            (('Non-integer swap', 256, 1, 10, 20, 'flavor1'), {'swap': -1}),
 
-            (('Negative rxtx_factor', 256, 1, 10, 'f1'), {'rxtx_factor': -1}),
+            (('Negative rxtx_factor', 256, 1, 10, 20, 'f1'),
-            (('Non-integer rxtx_factor', 256, 1, 10, 'f1'),
+                 {'rxtx_factor': -1}),
+            (('Non-integer rxtx_factor', 256, 1, 10, 20, 'f1'),
                  {'rxtx_factor': "d"}),
         ]
 
@@ -140,18 +142,18 @@ class InstanceTypeTestCase(test.TestCase):
     def test_duplicate_names_fail(self):
         """Ensures that name duplicates raise ApiError"""
         name = 'some_name'
-        instance_types.create(name, 256, 1, 120, 'flavor1')
+        instance_types.create(name, 256, 1, 120, 200, 'flavor1')
         self.assertRaises(exception.ApiError,
                           instance_types.create,
-                          name, "256", 1, 120, 'flavor2')
+                          name, "256", 1, 120, 200, 'flavor2')
 
     def test_duplicate_flavorids_fail(self):
         """Ensures that flavorid duplicates raise ApiError"""
         flavorid = 'flavor1'
-        instance_types.create('name one', 256, 1, 120, flavorid)
+        instance_types.create('name one', 256, 1, 120, 200, flavorid)
         self.assertRaises(exception.ApiError,
                           instance_types.create,
-                          'name two', 256, 1, 120, flavorid)
+                          'name two', 256, 1, 120, 200, flavorid)
 
     def test_will_not_destroy_with_no_name(self):
         """Ensure destroy sad path of no name raises error"""
@@ -239,14 +241,14 @@ class InstanceTypeFilteringTest(test.TestCase):
         expected = ['m1.large', 'm1.medium', 'm1.small', 'm1.xlarge']
         self.assertFilterResults(filters, expected)
 
-    def test_min_local_gb_filter(self):
+    def test_min_root_gb_filter(self):
         """Exclude everything but large and xlarge which have >= 80 GB"""
-        filters = dict(min_local_gb=80)
+        filters = dict(min_root_gb=80)
         expected = ['m1.large', 'm1.xlarge']
         self.assertFilterResults(filters, expected)
 
-    def test_min_memory_mb_AND_local_gb_filter(self):
+    def test_min_memory_mb_AND_root_gb_filter(self):
         """Exclude everything but large and xlarge which have >= 80 GB"""
-        filters = dict(min_memory_mb=16384, min_local_gb=80)
+        filters = dict(min_memory_mb=16384, min_root_gb=80)
         expected = ['m1.xlarge']
         self.assertFilterResults(filters, expected)

nova/tests/test_libvirt.py

@@ -303,7 +303,8 @@ class LibvirtConnTestCase(test.TestCase):
                      'project_id': 'fake',
                      'bridge': 'br101',
                      'image_ref': '155d900f-4e14-4e4c-a73d-069cbf4541e6',
-                     'local_gb': 20,
+                     'root_gb': 10,
+                     'ephemeral_gb': 20,
                      'instance_type_id': '5'}  # m1.small
 
     def create_fake_libvirt_mock(self, **kwargs):
@@ -1625,7 +1626,8 @@ class NWFilterTestCase(test.TestCase):
         inst['name'] = 'm1.small'
         inst['memory_mb'] = '1024'
         inst['vcpus'] = '1'
-        inst['local_gb'] = '20'
+        inst['root_gb'] = '10'
+        inst['ephemeral_gb'] = '20'
         inst['flavorid'] = '1'
         inst['swap'] = '2048'
         inst['rxtx_factor'] = 1

nova/tests/test_xenapi.py

@@ -107,7 +107,7 @@ class XenAPIVolumeTestCase(test.TestCase):
                   'image_ref': 1,
                   'kernel_id': 2,
                   'ramdisk_id': 3,
-                  'local_gb': 20,
+                  'root_gb': 20,
                   'instance_type_id': '3',  # m1.large
                   'os_type': 'linux',
                   'architecture': 'x86-64'}
@@ -414,7 +414,7 @@ class XenAPIVMTestCase(test.TestCase):
                       'image_ref': image_ref,
                       'kernel_id': kernel_id,
                       'ramdisk_id': ramdisk_id,
-                      'local_gb': 20,
+                      'root_gb': 20,
                       'instance_type_id': instance_type_id,
                       'os_type': os_type,
                       'hostname': hostname,
@@ -700,7 +700,7 @@ class XenAPIVMTestCase(test.TestCase):
             'image_ref': 1,
             'kernel_id': 2,
             'ramdisk_id': 3,
-            'local_gb': 20,
+            'root_gb': 20,
             'instance_type_id': '3',  # m1.large
             'os_type': 'linux',
             'architecture': 'x86-64'}
@@ -797,7 +797,7 @@ class XenAPIMigrateInstance(test.TestCase):
                   'image_ref': 1,
                   'kernel_id': None,
                   'ramdisk_id': None,
-                  'local_gb': 5,
+                  'root_gb': 5,
                   'instance_type_id': '3',  # m1.large
                   'os_type': 'linux',
                   'architecture': 'x86-64'}
@@ -917,9 +917,9 @@ class XenAPIMigrateInstance(test.TestCase):
         def fake_vdi_resize(*args, **kwargs):
             self.called = True
 
+        self.stubs.Set(vmops.VMOps, '_start', fake_vm_start)
         self.stubs.Set(stubs.FakeSessionForMigrationTests,
                        "VDI_resize_online", fake_vdi_resize)
-        self.stubs.Set(vmops.VMOps, '_start', fake_vm_start)
 
         stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests)
         stubs.stubout_loopingcall_start(self.stubs)
@@ -949,7 +949,7 @@ class XenAPIMigrateInstance(test.TestCase):
         tiny_type_id = \
                 instance_types.get_instance_type_by_name('m1.tiny')['id']
         self.instance_values.update({'instance_type_id': tiny_type_id,
-                                     'local_gb': 0})
+                                     'root_gb': 0})
         instance = db.instance_create(self.context, self.instance_values)
 
         def fake_vdi_resize(*args, **kwargs):
@@ -1157,7 +1157,7 @@ class XenAPIAutoDiskConfigTestCase(test.TestCase):
                   'image_ref': 1,
                   'kernel_id': 2,
                   'ramdisk_id': 3,
-                  'local_gb': 20,
+                  'root_gb': 20,
                   'instance_type_id': '3',  # m1.large
                   'os_type': 'linux',
                   'architecture': 'x86-64'}
@@ -1226,6 +1226,87 @@ class XenAPIAutoDiskConfigTestCase(test.TestCase):
         self.assertIsPartitionCalled(True)
 
 
+class XenAPIGenerateLocal(test.TestCase):
+    """Test generating of local disks, like swap and ephemeral"""
+    def setUp(self):
+        super(XenAPIGenerateLocal, self).setUp()
+        self.stubs = stubout.StubOutForTesting()
+        self.flags(target_host='127.0.0.1',
+                   xenapi_connection_url='test_url',
+                   xenapi_connection_password='test_pass',
+                   xenapi_generate_swap=True,
+                   firewall_driver='nova.virt.xenapi.firewall.'
+                                   'Dom0IptablesFirewallDriver')
+        stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests)
+        db_fakes.stub_out_db_instance_api(self.stubs)
+        xenapi_fake.reset()
+        self.conn = xenapi_conn.get_connection(False)
+
+        self.user_id = 'fake'
+        self.project_id = 'fake'
+
+        self.instance_values = {'id': 1,
+                  'project_id': self.project_id,
+                  'user_id': self.user_id,
+                  'image_ref': 1,
+                  'kernel_id': 2,
+                  'ramdisk_id': 3,
+                  'root_gb': 20,
+                  'instance_type_id': '3',  # m1.large
+                  'os_type': 'linux',
+                  'architecture': 'x86-64'}
+
+        self.context = context.RequestContext(self.user_id, self.project_id)
+
+        @classmethod
+        def fake_create_vbd(cls, session, vm_ref, vdi_ref, userdevice,
+                            bootable=True):
+            pass
+
+        self.stubs.Set(volume_utils.VolumeHelper,
+                       "create_vbd",
+                       fake_create_vbd)
+
+    def assertCalled(self, instance):
+        disk_image_type = vm_utils.ImageType.DISK_VHD
+        vm_ref = "blah"
+        first_vdi_ref = "blah"
+        vdis = ["blah"]
+
+        self.called = False
+        self.conn._vmops._attach_disks(instance, disk_image_type,
+                                       vm_ref, first_vdi_ref, vdis)
+        self.assertTrue(self.called)
+
+    def test_generate_swap(self):
+        """Test swap disk generation."""
+        instance = db.instance_create(self.context, self.instance_values)
+        instance = db.instance_update(self.context, instance['id'],
+                                      {'instance_type_id': 5})
+
+        @classmethod
+        def fake_generate_swap(cls, *args, **kwargs):
+            self.called = True
+        self.stubs.Set(vm_utils.VMHelper, 'generate_swap',
+                       fake_generate_swap)
+
+        self.assertCalled(instance)
+
+    def test_generate_ephemeral(self):
+        """Test ephemeral disk generation."""
+        instance = db.instance_create(self.context, self.instance_values)
+        instance = db.instance_update(self.context, instance['id'],
+                                      {'instance_type_id': 4})
+
+        @classmethod
+        def fake_generate_ephemeral(cls, *args):
+            self.called = True
+        self.stubs.Set(vm_utils.VMHelper, 'generate_ephemeral',
+                       fake_generate_ephemeral)
+
+        self.assertCalled(instance)
+
+
 class XenAPIBWUsageTestCase(test.TestCase):
     def setUp(self):
         super(XenAPIBWUsageTestCase, self).setUp()

nova/tests/vmwareapi/db_fakes.py

@@ -31,13 +31,13 @@ def stub_out_db_instance_api(stubs):
     """Stubs out the db API for creating Instances."""
 
     INSTANCE_TYPES = {
-        'm1.tiny': dict(memory_mb=512, vcpus=1, local_gb=0, flavorid=1),
+        'm1.tiny': dict(memory_mb=512, vcpus=1, root_gb=0, flavorid=1),
-        'm1.small': dict(memory_mb=2048, vcpus=1, local_gb=20, flavorid=2),
+        'm1.small': dict(memory_mb=2048, vcpus=1, root_gb=20, flavorid=2),
         'm1.medium':
-            dict(memory_mb=4096, vcpus=2, local_gb=40, flavorid=3),
+            dict(memory_mb=4096, vcpus=2, root_gb=40, flavorid=3),
-        'm1.large': dict(memory_mb=8192, vcpus=4, local_gb=80, flavorid=4),
+        'm1.large': dict(memory_mb=8192, vcpus=4, root_gb=80, flavorid=4),
         'm1.xlarge':
-            dict(memory_mb=16384, vcpus=8, local_gb=160, flavorid=5)}
+            dict(memory_mb=16384, vcpus=8, root_gb=160, flavorid=5)}
 
     class FakeModel(object):
         """Stubs out for model."""
@@ -76,7 +76,7 @@ def stub_out_db_instance_api(stubs):
             'memory_mb': type_data['memory_mb'],
             'vcpus': type_data['vcpus'],
             'mac_addresses': [{'address': values['mac_address']}],
-            'local_gb': type_data['local_gb'],
+            'root_gb': type_data['root_gb'],
             }
         return FakeModel(base_options)
 

nova/tests/xenapi/stubs.py

@@ -289,10 +289,15 @@ def stub_out_vm_methods(stubs):
     def fake_spawn_rescue(self, context, inst, network_info, image_meta):
         inst._rescue = False
 
+    @classmethod
+    def fake_generate_ephemeral(cls, *args):
+        pass
+
     stubs.Set(vmops.VMOps, "_shutdown", fake_shutdown)
     stubs.Set(vmops.VMOps, "_acquire_bootlock", fake_acquire_bootlock)
     stubs.Set(vmops.VMOps, "_release_bootlock", fake_release_bootlock)
     stubs.Set(vmops.VMOps, "spawn_rescue", fake_spawn_rescue)
+    stubs.Set(vm_utils.VMHelper, 'generate_ephemeral', fake_generate_ephemeral)
 
 
 class FakeSessionForVolumeTests(fake.SessionBase):
@@ -383,6 +388,10 @@ def stub_out_migration_methods(stubs):
     def fake_reset_network(*args, **kwargs):
         pass
 
+    @classmethod
+    def fake_generate_ephemeral(cls, *args):
+        pass
+
     stubs.Set(vmops.VMOps, '_destroy', fake_destroy)
     stubs.Set(vm_utils.VMHelper, 'scan_default_sr', fake_sr)
     stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_sr)
@@ -392,3 +401,4 @@ def stub_out_migration_methods(stubs):
     stubs.Set(vm_utils.VMHelper, 'get_sr_path', fake_get_sr_path)
     stubs.Set(vmops.VMOps, 'reset_network', fake_reset_network)
     stubs.Set(vmops.VMOps, '_shutdown', fake_shutdown)
+    stubs.Set(vm_utils.VMHelper, 'generate_ephemeral', fake_generate_ephemeral)