Add support for ephemeral storage

Add previously missing ephemeral storage support for nova-lxd.
This patch provides the infrastructure  to translate
the configuration of the instance to something
that LXD can understand and act upon.

The way that the ephemeral storage works is that block_device_info
dict and instance object. If an empeheral storage device
is needed then the storage deivce is created on the compute host
and bind mounted into the container when the container is started.
This is due to the contanier is running unprivileged and restricted
mount(8)

To come is restricting the ephemeral storage based on the LXD storage
backend on the compute host.

Update exist unit tests and adds unit new tests.

Change-Id: I38d48708b3c6fb450258e03a6106f47be3aeb998
Signed-off-by: Chuck Short <chuck.short@canonical.com>

nova/tests/unit/virt/lxd/stubs.py

@@ -38,7 +38,7 @@ class MockConf(mock.Mock):
         lxd_default = {
             'root_dir': '/fake/lxd/root',
             'timeout': 20,
-            'retry_interval': 2
+            'retry_interval': 2,
         }
         lxd_default.update(lxd_kwargs)
         self.lxd = mock.Mock(lxd_args, **lxd_default)

nova/tests/unit/virt/lxd/test_config.py

@@ -70,8 +70,10 @@ class LXDTestContainerConfig(test.NoDBTestCase):
         instance = stubs._fake_instance()
         network_info = fake_network.fake_get_instance_nw_info(self)
         mock_config_drive.return_value = False
+        block_device_info = mock.Mock()
 
-        container_profile = self.config.create_profile(instance, network_info)
+        container_profile = self.config.create_profile(
+            instance, network_info, block_device_info)
         self.assertEqual(container_profile['name'], instance.name)
         self.assertEqual(container_profile['config'],
                          {'boot.autostart': 'True',
@@ -96,8 +98,10 @@ class LXDTestContainerConfig(test.NoDBTestCase):
         """Verify the LXD profile without network configuration."""
         instance = stubs._fake_instance()
         mock_config_drive.return_value = False
+        block_device_info = mock.Mock()
 
-        container_profile = self.config.create_profile(instance, None)
+        container_profile = self.config.create_profile(
+            instance, None, block_device_info)
         self.assertEqual(len(container_profile['devices']), 1)
         self.assertEqual(container_profile['devices']['root'],
                          {'path': '/', 'size': '10GB', 'type': 'disk'})
@@ -109,7 +113,9 @@ class LXDTestContainerConfig(test.NoDBTestCase):
         """Verify the LXD profile with both network and configdrive enabled."""
         instance = stubs._fake_instance()
         network_info = fake_network.fake_get_instance_nw_info(self)
-        container_profile = self.config.create_profile(instance, network_info)
+        block_device_info = mock.Mock()
+        container_profile = self.config.create_profile(
+            instance, network_info, block_device_info)
         mock_configdrive.return_value = True
 
         self.assertEqual(len(container_profile['devices']), 3)
@@ -134,7 +140,9 @@ class LXDTestContainerConfig(test.NoDBTestCase):
     def test_create_contianer_profile_configdrive_network(self, mock_config):
         """Verify the LXD profile with no network and configdrive enabled."""
         instance = stubs._fake_instance()
-        container_profile = self.config.create_profile(instance, None)
+        block_device_info = mock.Mock()
+        container_profile = self.config.create_profile(
+            instance, None, block_device_info)
         mock_config.return_value = True
 
         self.assertEqual(len(container_profile['devices']), 2)
@@ -143,11 +151,90 @@ class LXDTestContainerConfig(test.NoDBTestCase):
                           'path': 'var/lib/cloud/data',
                           'source': '/fake/instances/path/'
                                     'instance-00000001/configdrive',
-                          'type': 'disk'},
+                          'type': 'disk'}
                          )
         self.assertEqual(container_profile['devices']['root'],
                          {'path': '/', 'size': '10GB', 'type': 'disk'})
 
+    @mock.patch.object(session.LXDAPISession, 'get_host_config',
+                       mock.Mock(return_value={'storage': 'zfs'}))
+    def test_create_container_profile_with_ephemeral_and_network(self):
+        """Verify the LXD profile with ephemeral drive is enabled."""
+        instance = stubs._fake_instance()
+        instance.ephemeral_gb = 1
+        network_info = fake_network.fake_get_instance_nw_info(self)
+        block_device_info = {'ephemerals': [{'virtual_name': 'ephemeral0'}]}
+        container_profile = self.config.create_profile(
+            instance, network_info, block_device_info)
+        self.assertEqual(len(container_profile['devices']), 3)
+        self.assertEqual(container_profile['devices']['root'],
+                         {'path': '/', 'size': '10GB', 'type': 'disk'})
+        self.assertEqual(container_profile['devices']['fake_br1'],
+                         {'hwaddr': 'DE:AD:BE:EF:00:01',
+                          'nictype': 'bridged',
+                          'parent': 'fake_br1',
+                          'type': 'nic'})
+        self.assertEqual(container_profile['devices']['ephemeral0'],
+                         {'optional': 'True',
+                          'path': '/mnt',
+                          'source': '/fake/instances/path/'
+                                    'instance-00000001/storage/ephemeral0',
+                          'type': 'disk'}
+                         )
+
+    @mock.patch.object(session.LXDAPISession, 'get_host_config',
+                       mock.Mock(return_value={'storage': 'zfs'}))
+    @mock.patch('nova.virt.configdrive.required_by')
+    def test_create_container_profile_with_ephemeral_and_configdrive(
+            self, mock_configdrive):
+        """Verify container profile configuration with ephemeral and
+           configdrive.
+        """
+        instance = stubs._fake_instance()
+        instance.ephemeral_gb = 1
+        network_info = fake_network.fake_get_instance_nw_info(self)
+        block_device_info = {'ephemerals': [{'virtual_name': 'ephemeral0'}]}
+        mock_configdrive.return_value = True
+        container_profile = self.config.create_profile(
+            instance, network_info, block_device_info)
+        self.assertEqual(len(container_profile['devices']), 4)
+        self.assertEqual(container_profile['devices']['root'],
+                         {'path': '/', 'size': '10GB', 'type': 'disk'})
+        self.assertEqual(container_profile['devices']['fake_br1'],
+                         {'hwaddr': 'DE:AD:BE:EF:00:01',
+                          'nictype': 'bridged',
+                          'parent': 'fake_br1',
+                          'type': 'nic'})
+        self.assertEqual(container_profile['devices']['ephemeral0'],
+                         {'optional': 'True',
+                          'path': '/mnt',
+                          'source': '/fake/instances/path/'
+                                    'instance-00000001/storage/ephemeral0',
+                          'type': 'disk'}
+                         )
+
+    @mock.patch.object(session.LXDAPISession, 'get_host_config',
+                       mock.Mock(return_value={'storage': 'zfs'}))
+    def test_create_container_profile_with_ephemeral_and_no_network(self):
+        """Verify container profile configuration with ephemeral and
+            without network.
+        """
+        instance = stubs._fake_instance()
+        instance.ephemeral_gb = 1
+        block_device_info = {'ephemerals': [{'virtual_name': 'ephemeral0'}]}
+        container_profile = self.config.create_profile(
+            instance, None, block_device_info)
+        self.assertEqual(len(container_profile['devices']), 2)
+        self.assertEqual(container_profile['devices']['root'],
+                         {'path': '/', 'size': '10GB', 'type': 'disk'})
+        self.assertEqual(container_profile['devices']['ephemeral0'],
+                         {'optional': 'True',
+                          'path': '/mnt',
+                          'source': '/fake/instances/path/'
+                          'instance-00000001/storage/ephemeral0',
+                          'type': 'disk'}
+                         )
+
     def test_create_network(self):
         instance = stubs._fake_instance()
         instance_name = 'fake_instance'

nova/tests/unit/virt/lxd/test_driver_api.py

@@ -230,7 +230,7 @@ class LXDTestDriver(test.NoDBTestCase):
                 context, instance, image_meta)
             mock_plug_vif.assert_called_once_with(instance, network_info)
             mock_container_profile.assert_called_once_with(
-                instance, network_info)
+                instance, network_info, block_device_info)
             mock_profile_create.assert_called_once_with({}, instance)
             mock_container_init.assert_called_once_with(container_config,
                                                         instance)
@@ -302,7 +302,7 @@ class LXDTestDriver(test.NoDBTestCase):
                 context, instance, image_meta)
             mock_plug_vif.assert_called_once_with(instance, network_info)
             mock_container_profile.assert_called_once_with(
-                instance, network_info)
+                instance, network_info, block_device_info)
             mock_profile_create.assert_called_once_with({}, instance)
             mock_configdrive.assert_called_once_with(instance)
             mock_add_configdrive.assert_called_once_with(

nova/virt/lxd/config.py

@@ -68,7 +68,7 @@ class LXDContainerConfig(object):
                           {'instance': instance_name, 'ex': ex},
                           instance=instance)
 
-    def create_profile(self, instance, network_info):
+    def create_profile(self, instance, network_info, block_device_info):
         """Create a LXD container profile configuration
 
         :param instance: nova instance object
@@ -86,10 +86,23 @@ class LXDContainerConfig(object):
             # Restrict the size of the "/" disk
             config['devices'] = self.configure_container_root(instance)
 
-            if network_info:
+            if instance.get('ephemeral_gb', 0) != 0:
-                config['devices'].update(self.create_network(instance_name,
+                ephemerals = block_device_info.get('ephemerals', [])
-                                                             instance,
+
-                                                             network_info))
+                if ephemerals == []:
+                    ephemeral_src = container_dir.get_container_storage(
+                        ephemerals['virtual_name'], instance.name)
+                    config['devices'].update(
+                        self.configure_disk_path(
+                            ephemeral_src, '/mnt', ephemerals['virtual_name'],
+                            instance))
+                else:
+                    for idx, ephemerals in enumerate(ephemerals):
+                        ephemeral_src = container_dir.get_container_storage(
+                            ephemerals['virtual_name'], instance.name)
+                        config['devices'].update(self.configure_disk_path(
+                            ephemeral_src, '/mnt', ephemerals['virtual_name'],
+                            instance))
 
             # if a configdrive is required, setup the mount point for
             # the container
@@ -102,6 +115,11 @@ class LXDContainerConfig(object):
                     'configdrive', instance)
                 config['devices'].update(config_drive)
 
+            if network_info:
+                config['devices'].update(self.create_network(instance_name,
+                                                             instance,
+                                                             network_info))
+
             return config
         except Exception as ex:
             with excutils.save_and_reraise_exception():
@@ -173,13 +191,9 @@ class LXDContainerConfig(object):
         try:
             config = {}
             lxd_config = self.session.get_host_config(instance)
+            config.setdefault('root', {'type': 'disk', 'path': '/'})
             if str(lxd_config['storage']) in ['btrfs', 'zfs']:
-                config['root'] = {'path': '/',
+                config['root'].update({'size': '%sGB' % str(instance.root_gb)})
-                                  'type': 'disk',
-                                  'size': '%sGB' % str(instance.root_gb)}
-            else:
-                config['root'] = {'path': '/',
-                                  'type': 'disk'}
 
             # Set disk quotas
             config['root'].update(self.create_disk_quota_config(instance))

nova/virt/lxd/driver.py

@@ -206,7 +206,8 @@ class LXDDriver(driver.ComputeDriver):
 
             # Create the container profile
             container_profile = self.config.create_profile(instance,
-                                                           network_info)
+                                                           network_info,
+                                                           block_device_info)
             self.session.profile_create(container_profile, instance)
 
             # Create the container
@@ -222,6 +223,8 @@ class LXDDriver(driver.ComputeDriver):
             if configdrive.required_by(instance):
                 self._add_configdrive(instance, injected_files)
 
+            self._add_ephemeral(block_device_info, instance)
+
             # Start the container
             self.session.container_start(instance_name, instance)
 
@@ -232,6 +235,27 @@ class LXDDriver(driver.ComputeDriver):
                           {'instance': instance.name, 'ex': ex},
                           instance=instance)
 
+    def _add_ephemeral(self, block_device_info, instance):
+        if instance.get('ephemeral_gb', 0) != 0:
+            ephemerals = block_device_info.get('ephemerals', [])
+
+            root_dir = container_utils.get_container_rootfs(instance.name)
+            if ephemerals == []:
+                ephemeral_src = container_utils.get_container_storage(
+                    ephemerals['virtual_name'], instance.name)
+                fileutils.ensure_tree(ephemeral_src)
+                utils.execute('chown',
+                              os.stat(root_dir).st_uid,
+                              ephemeral_src, run_as_root=True)
+            else:
+                for id, ephx in enumerate(ephemerals):
+                    ephemeral_src = container_utils.get_container_storage(
+                        ephx['virtual_name'], instance.name)
+                    fileutils.ensure_tree(ephemeral_src)
+                    utils.execute('chown',
+                                  os.stat(root_dir).st_uid,
+                                  ephemeral_src, run_as_root=True)
+
     def _add_configdrive(self, instance, injected_files):
         """Configure the config drive for the container
 
@@ -309,15 +333,11 @@ class LXDDriver(driver.ComputeDriver):
                 self.unplug_vifs(instance, network_info)
 
             name = pwd.getpwuid(os.getuid()).pw_name
-            configdrive_dir = \
-                container_utils.get_container_configdrive(instance.name)
-            if os.path.exists(configdrive_dir):
-                utils.execute('chown', '-R', '%s:%s' % (name, name),
-                              configdrive_dir, run_as_root=True)
-                shutil.rmtree(configdrive_dir)
 
             container_dir = container_utils.get_instance_dir(instance.name)
             if os.path.exists(container_dir):
+                utils.execute('chown', '-R', '%s:%s' % (name, name),
+                              container_dir, run_as_root=True)
                 shutil.rmtree(container_dir)
         except Exception as ex:
             with excutils.save_and_reraise_exception():

nova/virt/lxd/utils.py

@@ -34,6 +34,10 @@ def get_container_manifest_image(image_meta):
     return os.path.join(BASE_DIR, '%s-manifest.tar' % image_meta.id)
 
 
+def get_container_storage(ephemeral, instance):
+    return os.path.join(CONF.instances_path, instance, 'storage', ephemeral)
+
+
 def get_container_configdrive(instance):
     return os.path.join(CONF.instances_path, instance, 'configdrive')