From b03c33ce64e10b5393c7969e82b449dfbed131c4 Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Tue, 5 Jul 2016 10:55:27 -0400 Subject: [PATCH] Add support for ephemeral storage Add previously missing ephemeral storage support for nova-lxd. This patch provides the infrastructure to translate the configuration of the instance to something that LXD can understand and act upon. The way that the ephemeral storage works is that block_device_info dict and instance object. If an empeheral storage device is needed then the storage deivce is created on the compute host and bind mounted into the container when the container is started. This is due to the contanier is running unprivileged and restricted mount(8) To come is restricting the ephemeral storage based on the LXD storage backend on the compute host. Update exist unit tests and adds unit new tests. Change-Id: I38d48708b3c6fb450258e03a6106f47be3aeb998 Signed-off-by: Chuck Short --- nova/tests/unit/virt/lxd/stubs.py | 2 +- nova/tests/unit/virt/lxd/test_config.py | 97 +++++++++++++++++++-- nova/tests/unit/virt/lxd/test_driver_api.py | 4 +- nova/virt/lxd/config.py | 36 +++++--- nova/virt/lxd/driver.py | 34 ++++++-- nova/virt/lxd/utils.py | 4 + 6 files changed, 151 insertions(+), 26 deletions(-) diff --git a/nova/tests/unit/virt/lxd/stubs.py b/nova/tests/unit/virt/lxd/stubs.py index c66c061d..18313277 100644 --- a/nova/tests/unit/virt/lxd/stubs.py +++ b/nova/tests/unit/virt/lxd/stubs.py @@ -38,7 +38,7 @@ class MockConf(mock.Mock): lxd_default = { 'root_dir': '/fake/lxd/root', 'timeout': 20, - 'retry_interval': 2 + 'retry_interval': 2, } lxd_default.update(lxd_kwargs) self.lxd = mock.Mock(lxd_args, **lxd_default) diff --git a/nova/tests/unit/virt/lxd/test_config.py b/nova/tests/unit/virt/lxd/test_config.py index eb7f2338..da2d71c7 100644 --- a/nova/tests/unit/virt/lxd/test_config.py +++ b/nova/tests/unit/virt/lxd/test_config.py @@ -70,8 +70,10 @@ class LXDTestContainerConfig(test.NoDBTestCase): instance = stubs._fake_instance() network_info = fake_network.fake_get_instance_nw_info(self) mock_config_drive.return_value = False + block_device_info = mock.Mock() - container_profile = self.config.create_profile(instance, network_info) + container_profile = self.config.create_profile( + instance, network_info, block_device_info) self.assertEqual(container_profile['name'], instance.name) self.assertEqual(container_profile['config'], {'boot.autostart': 'True', @@ -96,8 +98,10 @@ class LXDTestContainerConfig(test.NoDBTestCase): """Verify the LXD profile without network configuration.""" instance = stubs._fake_instance() mock_config_drive.return_value = False + block_device_info = mock.Mock() - container_profile = self.config.create_profile(instance, None) + container_profile = self.config.create_profile( + instance, None, block_device_info) self.assertEqual(len(container_profile['devices']), 1) self.assertEqual(container_profile['devices']['root'], {'path': '/', 'size': '10GB', 'type': 'disk'}) @@ -109,7 +113,9 @@ class LXDTestContainerConfig(test.NoDBTestCase): """Verify the LXD profile with both network and configdrive enabled.""" instance = stubs._fake_instance() network_info = fake_network.fake_get_instance_nw_info(self) - container_profile = self.config.create_profile(instance, network_info) + block_device_info = mock.Mock() + container_profile = self.config.create_profile( + instance, network_info, block_device_info) mock_configdrive.return_value = True self.assertEqual(len(container_profile['devices']), 3) @@ -134,7 +140,9 @@ class LXDTestContainerConfig(test.NoDBTestCase): def test_create_contianer_profile_configdrive_network(self, mock_config): """Verify the LXD profile with no network and configdrive enabled.""" instance = stubs._fake_instance() - container_profile = self.config.create_profile(instance, None) + block_device_info = mock.Mock() + container_profile = self.config.create_profile( + instance, None, block_device_info) mock_config.return_value = True self.assertEqual(len(container_profile['devices']), 2) @@ -143,11 +151,90 @@ class LXDTestContainerConfig(test.NoDBTestCase): 'path': 'var/lib/cloud/data', 'source': '/fake/instances/path/' 'instance-00000001/configdrive', - 'type': 'disk'}, + 'type': 'disk'} ) self.assertEqual(container_profile['devices']['root'], {'path': '/', 'size': '10GB', 'type': 'disk'}) + @mock.patch.object(session.LXDAPISession, 'get_host_config', + mock.Mock(return_value={'storage': 'zfs'})) + def test_create_container_profile_with_ephemeral_and_network(self): + """Verify the LXD profile with ephemeral drive is enabled.""" + instance = stubs._fake_instance() + instance.ephemeral_gb = 1 + network_info = fake_network.fake_get_instance_nw_info(self) + block_device_info = {'ephemerals': [{'virtual_name': 'ephemeral0'}]} + container_profile = self.config.create_profile( + instance, network_info, block_device_info) + self.assertEqual(len(container_profile['devices']), 3) + self.assertEqual(container_profile['devices']['root'], + {'path': '/', 'size': '10GB', 'type': 'disk'}) + self.assertEqual(container_profile['devices']['fake_br1'], + {'hwaddr': 'DE:AD:BE:EF:00:01', + 'nictype': 'bridged', + 'parent': 'fake_br1', + 'type': 'nic'}) + self.assertEqual(container_profile['devices']['ephemeral0'], + {'optional': 'True', + 'path': '/mnt', + 'source': '/fake/instances/path/' + 'instance-00000001/storage/ephemeral0', + 'type': 'disk'} + ) + + @mock.patch.object(session.LXDAPISession, 'get_host_config', + mock.Mock(return_value={'storage': 'zfs'})) + @mock.patch('nova.virt.configdrive.required_by') + def test_create_container_profile_with_ephemeral_and_configdrive( + self, mock_configdrive): + """Verify container profile configuration with ephemeral and + configdrive. + """ + instance = stubs._fake_instance() + instance.ephemeral_gb = 1 + network_info = fake_network.fake_get_instance_nw_info(self) + block_device_info = {'ephemerals': [{'virtual_name': 'ephemeral0'}]} + mock_configdrive.return_value = True + container_profile = self.config.create_profile( + instance, network_info, block_device_info) + self.assertEqual(len(container_profile['devices']), 4) + self.assertEqual(container_profile['devices']['root'], + {'path': '/', 'size': '10GB', 'type': 'disk'}) + self.assertEqual(container_profile['devices']['fake_br1'], + {'hwaddr': 'DE:AD:BE:EF:00:01', + 'nictype': 'bridged', + 'parent': 'fake_br1', + 'type': 'nic'}) + self.assertEqual(container_profile['devices']['ephemeral0'], + {'optional': 'True', + 'path': '/mnt', + 'source': '/fake/instances/path/' + 'instance-00000001/storage/ephemeral0', + 'type': 'disk'} + ) + + @mock.patch.object(session.LXDAPISession, 'get_host_config', + mock.Mock(return_value={'storage': 'zfs'})) + def test_create_container_profile_with_ephemeral_and_no_network(self): + """Verify container profile configuration with ephemeral and + without network. + """ + instance = stubs._fake_instance() + instance.ephemeral_gb = 1 + block_device_info = {'ephemerals': [{'virtual_name': 'ephemeral0'}]} + container_profile = self.config.create_profile( + instance, None, block_device_info) + self.assertEqual(len(container_profile['devices']), 2) + self.assertEqual(container_profile['devices']['root'], + {'path': '/', 'size': '10GB', 'type': 'disk'}) + self.assertEqual(container_profile['devices']['ephemeral0'], + {'optional': 'True', + 'path': '/mnt', + 'source': '/fake/instances/path/' + 'instance-00000001/storage/ephemeral0', + 'type': 'disk'} + ) + def test_create_network(self): instance = stubs._fake_instance() instance_name = 'fake_instance' diff --git a/nova/tests/unit/virt/lxd/test_driver_api.py b/nova/tests/unit/virt/lxd/test_driver_api.py index aa7a44e4..5b9c0772 100644 --- a/nova/tests/unit/virt/lxd/test_driver_api.py +++ b/nova/tests/unit/virt/lxd/test_driver_api.py @@ -230,7 +230,7 @@ class LXDTestDriver(test.NoDBTestCase): context, instance, image_meta) mock_plug_vif.assert_called_once_with(instance, network_info) mock_container_profile.assert_called_once_with( - instance, network_info) + instance, network_info, block_device_info) mock_profile_create.assert_called_once_with({}, instance) mock_container_init.assert_called_once_with(container_config, instance) @@ -302,7 +302,7 @@ class LXDTestDriver(test.NoDBTestCase): context, instance, image_meta) mock_plug_vif.assert_called_once_with(instance, network_info) mock_container_profile.assert_called_once_with( - instance, network_info) + instance, network_info, block_device_info) mock_profile_create.assert_called_once_with({}, instance) mock_configdrive.assert_called_once_with(instance) mock_add_configdrive.assert_called_once_with( diff --git a/nova/virt/lxd/config.py b/nova/virt/lxd/config.py index 4780598c..5c56a158 100644 --- a/nova/virt/lxd/config.py +++ b/nova/virt/lxd/config.py @@ -68,7 +68,7 @@ class LXDContainerConfig(object): {'instance': instance_name, 'ex': ex}, instance=instance) - def create_profile(self, instance, network_info): + def create_profile(self, instance, network_info, block_device_info): """Create a LXD container profile configuration :param instance: nova instance object @@ -86,10 +86,23 @@ class LXDContainerConfig(object): # Restrict the size of the "/" disk config['devices'] = self.configure_container_root(instance) - if network_info: - config['devices'].update(self.create_network(instance_name, - instance, - network_info)) + if instance.get('ephemeral_gb', 0) != 0: + ephemerals = block_device_info.get('ephemerals', []) + + if ephemerals == []: + ephemeral_src = container_dir.get_container_storage( + ephemerals['virtual_name'], instance.name) + config['devices'].update( + self.configure_disk_path( + ephemeral_src, '/mnt', ephemerals['virtual_name'], + instance)) + else: + for idx, ephemerals in enumerate(ephemerals): + ephemeral_src = container_dir.get_container_storage( + ephemerals['virtual_name'], instance.name) + config['devices'].update(self.configure_disk_path( + ephemeral_src, '/mnt', ephemerals['virtual_name'], + instance)) # if a configdrive is required, setup the mount point for # the container @@ -102,6 +115,11 @@ class LXDContainerConfig(object): 'configdrive', instance) config['devices'].update(config_drive) + if network_info: + config['devices'].update(self.create_network(instance_name, + instance, + network_info)) + return config except Exception as ex: with excutils.save_and_reraise_exception(): @@ -173,13 +191,9 @@ class LXDContainerConfig(object): try: config = {} lxd_config = self.session.get_host_config(instance) + config.setdefault('root', {'type': 'disk', 'path': '/'}) if str(lxd_config['storage']) in ['btrfs', 'zfs']: - config['root'] = {'path': '/', - 'type': 'disk', - 'size': '%sGB' % str(instance.root_gb)} - else: - config['root'] = {'path': '/', - 'type': 'disk'} + config['root'].update({'size': '%sGB' % str(instance.root_gb)}) # Set disk quotas config['root'].update(self.create_disk_quota_config(instance)) diff --git a/nova/virt/lxd/driver.py b/nova/virt/lxd/driver.py index 260ac7d6..d8a23bec 100644 --- a/nova/virt/lxd/driver.py +++ b/nova/virt/lxd/driver.py @@ -206,7 +206,8 @@ class LXDDriver(driver.ComputeDriver): # Create the container profile container_profile = self.config.create_profile(instance, - network_info) + network_info, + block_device_info) self.session.profile_create(container_profile, instance) # Create the container @@ -222,6 +223,8 @@ class LXDDriver(driver.ComputeDriver): if configdrive.required_by(instance): self._add_configdrive(instance, injected_files) + self._add_ephemeral(block_device_info, instance) + # Start the container self.session.container_start(instance_name, instance) @@ -232,6 +235,27 @@ class LXDDriver(driver.ComputeDriver): {'instance': instance.name, 'ex': ex}, instance=instance) + def _add_ephemeral(self, block_device_info, instance): + if instance.get('ephemeral_gb', 0) != 0: + ephemerals = block_device_info.get('ephemerals', []) + + root_dir = container_utils.get_container_rootfs(instance.name) + if ephemerals == []: + ephemeral_src = container_utils.get_container_storage( + ephemerals['virtual_name'], instance.name) + fileutils.ensure_tree(ephemeral_src) + utils.execute('chown', + os.stat(root_dir).st_uid, + ephemeral_src, run_as_root=True) + else: + for id, ephx in enumerate(ephemerals): + ephemeral_src = container_utils.get_container_storage( + ephx['virtual_name'], instance.name) + fileutils.ensure_tree(ephemeral_src) + utils.execute('chown', + os.stat(root_dir).st_uid, + ephemeral_src, run_as_root=True) + def _add_configdrive(self, instance, injected_files): """Configure the config drive for the container @@ -309,15 +333,11 @@ class LXDDriver(driver.ComputeDriver): self.unplug_vifs(instance, network_info) name = pwd.getpwuid(os.getuid()).pw_name - configdrive_dir = \ - container_utils.get_container_configdrive(instance.name) - if os.path.exists(configdrive_dir): - utils.execute('chown', '-R', '%s:%s' % (name, name), - configdrive_dir, run_as_root=True) - shutil.rmtree(configdrive_dir) container_dir = container_utils.get_instance_dir(instance.name) if os.path.exists(container_dir): + utils.execute('chown', '-R', '%s:%s' % (name, name), + container_dir, run_as_root=True) shutil.rmtree(container_dir) except Exception as ex: with excutils.save_and_reraise_exception(): diff --git a/nova/virt/lxd/utils.py b/nova/virt/lxd/utils.py index 4aca0400..2b26d97e 100644 --- a/nova/virt/lxd/utils.py +++ b/nova/virt/lxd/utils.py @@ -34,6 +34,10 @@ def get_container_manifest_image(image_meta): return os.path.join(BASE_DIR, '%s-manifest.tar' % image_meta.id) +def get_container_storage(ephemeral, instance): + return os.path.join(CONF.instances_path, instance, 'storage', ephemeral) + + def get_container_configdrive(instance): return os.path.join(CONF.instances_path, instance, 'configdrive')