From af2e40362594d19fc4ca2be0a11c186b684946af Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 15 Sep 2021 15:49:40 +0100 Subject: [PATCH] Block nova-compute startup on mountpoint If an ephemeral-device storage configuration has been provided, ensure that the nova-compute service will not start until the mountpoint (currently /var/lib/nova/instances) has actually been mounted. If this does not happen the nova-compute service will fail to start in a failsafe condition. Change-Id: Ic16691e119e430faec9994f6e207596629e47bb6 Closes-Bug: 1863358 --- hooks/nova_compute_utils.py | 24 ++++++++++++++++++- templates/99-mount.conf | 3 +++ unit_tests/test_nova_compute_utils.py | 34 ++++++++++++++++++++++++--- 3 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 templates/99-mount.conf diff --git a/hooks/nova_compute_utils.py b/hooks/nova_compute_utils.py index cebcbcc7..2ec53208 100644 --- a/hooks/nova_compute_utils.py +++ b/hooks/nova_compute_utils.py @@ -123,6 +123,8 @@ from charmhelpers.contrib.storage.linux.utils import ( mkfs_xfs, ) +from charmhelpers.core.templating import render + CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' TEMPLATES = 'templates/' @@ -186,6 +188,9 @@ NOVA_COMPUTE_AA_PROFILE_PATH = ('/etc/apparmor.d/{}' NOVA_NETWORK_AA_PROFILE_PATH = ('/etc/apparmor.d/{}' ''.format(NOVA_NETWORK_AA_PROFILE)) +NOVA_COMPUTE_OVERRIDE_DIR = '/etc/systemd/system/nova-compute.service.d' +MOUNT_DEPENDENCY_OVERRIDE = '99-mount.conf' + LIBVIRT_TYPES = ['kvm', 'qemu', 'lxc'] USE_FQDN_KEY = 'nova-compute-charm-use-fqdn' @@ -1086,11 +1091,18 @@ def configure_local_ephemeral_storage(): level=DEBUG) return + mountpoint = '/var/lib/nova/instances' + db = kv() storage_configured = db.get('storage-configured', False) if storage_configured: log("Ephemeral storage already configured, skipping", level=DEBUG) + # NOTE(jamespage): + # Install mountpoint override to ensure that upgrades + # to the charm version which supports this change + # also start exhibiting the correct behaviour + install_mount_override(mountpoint) return dev = determine_block_device() @@ -1132,10 +1144,10 @@ def configure_local_ephemeral_storage(): # If not cleaned and in use, mkfs should fail. mkfs_xfs(dev, force=True) - mountpoint = '/var/lib/nova/instances' filesystem = "xfs" mount(dev, mountpoint, filesystem=filesystem) fstab_add(dev, mountpoint, filesystem, options=options) + install_mount_override(mountpoint) check_call(['chown', '-R', 'nova:nova', mountpoint]) check_call(['chmod', '-R', '0755', mountpoint]) @@ -1146,6 +1158,16 @@ def configure_local_ephemeral_storage(): db.flush() +def install_mount_override(mountpoint): + """Install override for nova-compute for configured mountpoint""" + render( + MOUNT_DEPENDENCY_OVERRIDE, + os.path.join(NOVA_COMPUTE_OVERRIDE_DIR, MOUNT_DEPENDENCY_OVERRIDE), + {'mount_point': mountpoint.replace('/', '-')[1:]}, + perms=0o644, + ) + + def get_availability_zone(): use_juju_az = config('customize-failure-domain') juju_az = os.environ.get('JUJU_AVAILABILITY_ZONE') diff --git a/templates/99-mount.conf b/templates/99-mount.conf new file mode 100644 index 00000000..6836a5f3 --- /dev/null +++ b/templates/99-mount.conf @@ -0,0 +1,3 @@ +[Unit] +Requires={{ mount_point }}.mount +After={{ mount_point }}.mount \ No newline at end of file diff --git a/unit_tests/test_nova_compute_utils.py b/unit_tests/test_nova_compute_utils.py index 92f0d0ca..c00e4cfc 100644 --- a/unit_tests/test_nova_compute_utils.py +++ b/unit_tests/test_nova_compute_utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import tempfile import nova_compute_context as compute_context @@ -996,6 +997,7 @@ class NovaComputeUtilsTests(CharmTestCase): self.config.assert_called_with('ephemeral-device') self.storage_list.assert_called_with('ephemeral-device') + @patch.object(utils, 'install_mount_override') @patch.object(utils, 'filter_installed_packages') @patch.object(utils, 'uuid') @patch.object(utils, 'determine_block_device') @@ -1003,7 +1005,8 @@ class NovaComputeUtilsTests(CharmTestCase): self, determine_block_device, uuid, - filter_installed_packages): + filter_installed_packages, + install_mount_override): filter_installed_packages.return_value = [] determine_block_device.return_value = '/dev/sdb' uuid.uuid4.return_value = 'test' @@ -1043,17 +1046,22 @@ class NovaComputeUtilsTests(CharmTestCase): 'x-systemd.requires=vaultlocker-decrypt@test.service,' 'comment=vaultlocker' ) + install_mount_override.assert_called_with( + '/var/lib/nova/instances' + ) self.assertTrue(self.test_kv.get('storage-configured')) self.vaultlocker.write_vaultlocker_conf.assert_called_with( 'test_context', priority=80 ) + @patch.object(utils, 'install_mount_override') @patch.object(utils, 'uuid') @patch.object(utils, 'determine_block_device') def test_configure_local_ephemeral_storage(self, determine_block_device, - uuid): + uuid, + install_mount_override): determine_block_device.return_value = '/dev/sdb' uuid.uuid4.return_value = 'test' @@ -1088,12 +1096,17 @@ class NovaComputeUtilsTests(CharmTestCase): 'xfs', options=None ) + install_mount_override.assert_called_with( + '/var/lib/nova/instances' + ) self.assertTrue(self.test_kv.get('storage-configured')) self.vaultlocker.write_vaultlocker_conf.assert_not_called() + @patch.object(utils, 'install_mount_override') @patch.object(utils, 'filter_installed_packages') def test_configure_local_ephemeral_storage_done(self, - filter_installed_packages): + filter_installed_packages, + install_mount_override): filter_installed_packages.return_value = [] self.test_kv.set('storage-configured', True) @@ -1113,6 +1126,10 @@ class NovaComputeUtilsTests(CharmTestCase): priority=80 ) self.is_block_device.assert_not_called() + # NOTE: called to deal with charm upgrades + install_mount_override.assert_called_with( + '/var/lib/nova/instances' + ) @patch.object(utils.os.environ, 'get') def test_get_az_customize_with_env(self, os_environ_get_mock): @@ -1190,3 +1207,14 @@ class NovaComputeUtilsTests(CharmTestCase): self.assertEquals(utils.use_fqdn_hint(), False) _kv().get.return_value = True self.assertEquals(utils.use_fqdn_hint(), True) + + @patch.object(utils, 'render') + def test_install_mount_override(self, render): + utils.install_mount_override('/srv/test') + render.assert_called_once_with( + utils.MOUNT_DEPENDENCY_OVERRIDE, + os.path.join(utils.NOVA_COMPUTE_OVERRIDE_DIR, + utils.MOUNT_DEPENDENCY_OVERRIDE), + {'mount_point': 'srv-test'}, + perms=0o644, + )