From d507bd9f186c45b51d635e26070aef7280b6c175 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Sun, 5 Aug 2012 21:35:28 +1000 Subject: [PATCH] Config drive v2 This is the first cut of config drive v2. Some points to note: - implements a helper to create new-style config drives. These config drives can be VFAT or ISO9660, this is controlled by a flag. The current default is ISO9660. - the config drives contain all the injected files, as well as everything returned from the ec2 style metadata service. Only the most recent version of the ec2 metadata is used, but future versions will appear as well. - the v1 functionality of specifying an image from glance to have the files injected into is dropped. - the location for file injection is now a directory named openstack/files, not the root level of the filesystem. Filename mapping is in the openstack metadata files. - the default format for the config drive is iso9660, although the previous vfat is available with a flag change. - includes the first version of an openstack metadata format. - there are some simple unit tests which probably need more done to them. Partially implements bp config-drive-v2. Change-Id: I210fa4dd7d8d6be398a46b30a0d46b960e22d6b0 --- etc/nova/rootwrap.d/compute.filters | 2 + nova/api/metadata/base.py | 46 ++++++++ nova/exception.py | 10 ++ nova/tests/test_configdrive2.py | 103 +++++++++++++++++ nova/tests/test_virt_drivers.py | 8 ++ nova/virt/configdrive.py | 173 ++++++++++++++++++++++++++++ nova/virt/libvirt/driver.py | 102 ++++++++-------- 7 files changed, 396 insertions(+), 48 deletions(-) create mode 100644 nova/tests/test_configdrive2.py create mode 100644 nova/virt/configdrive.py diff --git a/etc/nova/rootwrap.d/compute.filters b/etc/nova/rootwrap.d/compute.filters index 7e6191ce9cda..607e7376c2ca 100644 --- a/etc/nova/rootwrap.d/compute.filters +++ b/etc/nova/rootwrap.d/compute.filters @@ -13,10 +13,12 @@ tune2fs: CommandFilter, /sbin/tune2fs, root # nova/virt/disk/mount.py: 'mount', mapped_device, mount_dir # nova/virt/xenapi/vm_utils.py: 'mount', '-t', 'ext2,ext3,ext4,reiserfs'.. +# nova/virt/configdrive.py: 'mount', device, mountdir mount: CommandFilter, /bin/mount, root # nova/virt/disk/mount.py: 'umount', mapped_device # nova/virt/xenapi/vm_utils.py: 'umount', dev_path +# nova/virt/configdrive.py: 'umount', mountdir umount: CommandFilter, /bin/umount, root # nova/virt/disk/nbd.py: 'qemu-nbd', '-c', device, image diff --git a/nova/api/metadata/base.py b/nova/api/metadata/base.py index 61bc9cad00d5..78ebdd82170f 100644 --- a/nova/api/metadata/base.py +++ b/nova/api/metadata/base.py @@ -19,6 +19,7 @@ """Instance Metadata information.""" import base64 +import json import os from nova.api.ec2 import ec2utils @@ -27,9 +28,21 @@ from nova import context from nova import db from nova import flags from nova import network +from nova.openstack.common import cfg + + +metadata_opts = [ + cfg.StrOpt('config_drive_skip_versions', + default=('1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 ' + '2007-12-15 2008-02-01 2008-09-01'), + help=('List of metadata versions to skip placing into the ' + 'config drive')), + ] FLAGS = flags.FLAGS flags.DECLARE('dhcp_domain', 'nova.network.manager') +FLAGS.register_opts(metadata_opts) + _DEFAULT_MAPPINGS = {'ami': 'sda1', 'ephemeral0': 'sda2', @@ -205,6 +218,39 @@ class InstanceMetadata(): return data + def metadata_for_config_drive(self, injected_files): + """Yields (path, value) tuples for metadata elements.""" + # EC2 style metadata + for version in VERSIONS: + if version in FLAGS.config_drive_skip_versions.split(' '): + continue + + data = self.get_ec2_metadata(version) + if 'user-data' in data: + filepath = os.path.join('ec2', version, 'userdata.raw') + yield (filepath, data['user-data']) + del data['user-data'] + + try: + del data['public-keys']['0']['_name'] + except KeyError: + pass + + filepath = os.path.join('ec2', version, 'metadata.json') + yield (filepath, json.dumps(data['meta-data'])) + + filepath = os.path.join('ec2', 'latest', 'metadata.json') + yield (filepath, json.dumps(data['meta-data'])) + + # Openstack style metadata + # TODO(mikal): refactor this later + files = [] + for path in injected_files: + files.append({'path': path, + 'content': injected_files[path]}) + yield ('openstack/2012-08-10/files.json', json.dumps(files)) + yield ('openstack/latest/files.json', json.dumps(files)) + def get_metadata_by_address(address): ctxt = context.get_admin_context() diff --git a/nova/exception.py b/nova/exception.py index 5febfb92c8cd..315147764ec2 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -1145,6 +1145,16 @@ class InstanceIsLocked(InstanceInvalidState): message = _("Instance %(instance_uuid)s is locked") +class ConfigDriveMountFailed(NovaException): + message = _("Could not mount vfat config drive. %(operation)s failed. " + "Error: %(error)s") + + +class ConfigDriveUnknownFormat(NovaException): + message = _("Unknown config drive format %(format)s. Select one of " + "iso9660 or vfat.") + + def get_context_from_function_and_args(function, args, kwargs): """Find an arg of type RequestContext and return it. diff --git a/nova/tests/test_configdrive2.py b/nova/tests/test_configdrive2.py new file mode 100644 index 000000000000..b9467f2586a5 --- /dev/null +++ b/nova/tests/test_configdrive2.py @@ -0,0 +1,103 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Michael Still and Canonical Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import mox +import os +import subprocess +import tempfile + +from nova import test + +from nova import flags +from nova.openstack.common import log +from nova import utils +from nova.virt import configdrive +from nova.virt.libvirt import utils as virtutils + + +FLAGS = flags.FLAGS + +LOG = log.getLogger(__name__) + + +class ConfigDriveTestCase(test.TestCase): + + def test_create_configdrive_iso(self): + imagefile = None + + try: + self.mox.StubOutWithMock(utils, 'execute') + + utils.execute('genisoimage', '-o', mox.IgnoreArg(), '-ldots', + '-allow-lowercase', '-allow-multidot', '-l', + '-publisher', mox.IgnoreArg(), '-quiet', '-J', '-r', + '-V', 'config-2', mox.IgnoreArg(), attempts=1, + run_as_root=False).AndReturn(None) + + self.mox.ReplayAll() + + c = configdrive.ConfigDriveBuilder() + c._add_file('this/is/a/path/hello', 'This is some content') + (fd, imagefile) = tempfile.mkstemp(prefix='cd_iso_') + os.close(fd) + c._make_iso9660(imagefile) + c.cleanup() + + # Check cleanup + self.assertFalse(os.path.exists(c.tempdir)) + + finally: + if imagefile: + utils.delete_if_exists(imagefile) + + def test_create_configdrive_vfat(self): + imagefile = None + try: + self.mox.StubOutWithMock(virtutils, 'mkfs') + self.mox.StubOutWithMock(utils, 'execute') + self.mox.StubOutWithMock(utils, 'trycmd') + + virtutils.mkfs('vfat', mox.IgnoreArg(), + label='config-2').AndReturn(None) + utils.trycmd('mount', '-o', 'loop', mox.IgnoreArg(), + mox.IgnoreArg(), + run_as_root=True).AndReturn((None, None)) + utils.trycmd('chown', mox.IgnoreArg(), mox.IgnoreArg(), + run_as_root=True).AndReturn((None, None)) + utils.execute('umount', mox.IgnoreArg(), + run_as_root=True).AndReturn(None) + + self.mox.ReplayAll() + + c = configdrive.ConfigDriveBuilder() + c._add_file('this/is/a/path/hello', 'This is some content') + (fd, imagefile) = tempfile.mkstemp(prefix='cd_vfat_') + os.close(fd) + c._make_vfat(imagefile) + c.cleanup() + + # Check cleanup + self.assertFalse(os.path.exists(c.tempdir)) + + # NOTE(mikal): we can't check for a VFAT output here because the + # filesystem creation stuff has been mocked out because it + # requires root permissions + + finally: + if imagefile: + utils.delete_if_exists(imagefile) diff --git a/nova/tests/test_virt_drivers.py b/nova/tests/test_virt_drivers.py index f42de66e538f..68b4fe4f2fe8 100644 --- a/nova/tests/test_virt_drivers.py +++ b/nova/tests/test_virt_drivers.py @@ -88,6 +88,9 @@ class _FakeDriverBackendTestCase(test.TestCase): def fake_migrateToURI(*a): pass + def fake_make_drive(_self, _path): + pass + self.stubs.Set(nova.virt.libvirt.driver.disk, 'extend', fake_extend) @@ -96,6 +99,11 @@ class _FakeDriverBackendTestCase(test.TestCase): self.stubs.Set(nova.virt.libvirt.driver.libvirt.Domain, 'migrateToURI', fake_migrateToURI) + # We can't actually make a config drive v2 because ensure_tree has + # been faked out + self.stubs.Set(nova.virt.configdrive.ConfigDriveBuilder, + 'make_drive', fake_make_drive) + def _teardown_fakelibvirt(self): # Restore libvirt import nova.virt.libvirt.driver diff --git a/nova/virt/configdrive.py b/nova/virt/configdrive.py new file mode 100644 index 000000000000..bf86f6f812e6 --- /dev/null +++ b/nova/virt/configdrive.py @@ -0,0 +1,173 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Michael Still and Canonical Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Config Drive v2 helper.""" + +import base64 +import json +import os +import shutil +import tempfile + +from nova.api.metadata import base as instance_metadata +from nova import exception +from nova import flags +from nova.openstack.common import cfg +from nova.openstack.common import log as logging +from nova import utils +from nova import version +from nova.virt.libvirt import utils as virtutils + +LOG = logging.getLogger(__name__) + +configdrive_opts = [ + cfg.StrOpt('config_drive_format', + default='iso9660', + help='Config drive format. One of iso9660 (default) or vfat'), + cfg.StrOpt('config_drive_tempdir', + default=tempfile.tempdir, + help=('Where to put temporary files associated with ' + 'config drive creation')), + ] + +FLAGS = flags.FLAGS +FLAGS.register_opts(configdrive_opts) + + +class ConfigDriveBuilder(object): + def __init__(self, instance=None): + self.instance = instance + self.imagefile = None + self.injected = {} + self.next_inject_id = 1 + + # TODO(mikal): I don't think I can use utils.tempdir here, because + # I need to have the directory last longer than the scope of this + # method call + self.tempdir = tempfile.mkdtemp(dir=FLAGS.config_drive_tempdir, + prefix='cd_gen_') + + def _add_file(self, path, data, inject=False): + if inject: + path_id = '%03d' % self.next_inject_id + path = 'openstack/files/%s' % path_id + self.injected[path] = path_id + self.next_inject_id += 1 + + filepath = os.path.join(self.tempdir, path) + dirname = os.path.dirname(filepath) + virtutils.ensure_tree(dirname) + with open(filepath, 'w') as f: + f.write(data) + + def add_instance_metadata(self): + inst_md = instance_metadata.InstanceMetadata(self.instance) + for (path, value) in inst_md.metadata_for_config_drive(self.injected): + self._add_file(path, value) + LOG.debug(_('Added %(filepath)s to config drive'), + {'filepath': path}, instance=self.instance) + + def inject_data(self, key, net, metadata, admin_pass, files): + if key: + self._add_file('key', key, inject=True) + if net: + self._add_file('net', net, inject=True) + if metadata: + self._add_file('metadata', metadata, inject=True) + if admin_pass: + self._add_file('adminpass', admin_pass, inject=True) + if files: + files_struct = [] + for (path, contents) in files: + files.append[{'path': path, + 'encoding': 'base64', + 'data': base64.b64encode(contents), + }] + self._add_file('files', json.dumps(files_struct), inject=True) + + def _make_iso9660(self, path): + utils.execute('genisoimage', + '-o', path, + '-ldots', + '-allow-lowercase', + '-allow-multidot', + '-l', + '-publisher', ('"OpenStack nova %s"' + % version.version_string()), + '-quiet', + '-J', + '-r', + '-V', 'config-2', + self.tempdir, + attempts=1, + run_as_root=False) + + def _make_vfat(self, path): + # NOTE(mikal): This is a little horrible, but I couldn't find an + # equivalent to genisoimage for vfat filesystems. vfat images are + # always 64mb. + with open(path, 'w') as f: + f.truncate(64 * 1024 * 1024) + + virtutils.mkfs('vfat', path, label='config-2') + + mounted = False + try: + mountdir = tempfile.mkdtemp(dir=FLAGS.config_drive_tempdir, + prefix='cd_mnt_') + _out, err = utils.trycmd('mount', '-o', 'loop', path, mountdir, + run_as_root=True) + if err: + raise exception.ConfigDriveMountFailed(operation='mount', + error=err) + mounted = True + + _out, err = utils.trycmd('chown', + '%s.%s' % (os.getuid(), os.getgid()), + mountdir, run_as_root=True) + if err: + raise exception.ConfigDriveMountFailed(operation='chown', + error=err) + + # NOTE(mikal): I can't just use shutils.copytree here, because the + # destination directory already exists. This is annoying. + for ent in os.listdir(self.tempdir): + shutil.copytree(os.path.join(self.tempdir, ent), + os.path.join(mountdir, ent)) + + finally: + if mounted: + utils.execute('umount', mountdir, run_as_root=True) + shutil.rmtree(mountdir) + + def make_drive(self, path): + if FLAGS.config_drive_format == 'iso9660': + self._make_iso9660(path) + elif FLAGS.config_drive_format == 'vfat': + self._make_vfat(path) + else: + raise exception.ConfigDriveUnknownFormat( + format=FLAGS.config_drive_format) + + def cleanup(self): + if self.imagefile: + utils.delete_if_exists(self.imagefile) + + try: + shutil.rmtree(self.tempdir) + except OSError, e: + LOG.error(_('Could not remove tmpdir: %s'), str(e)) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index ba04dd70d3f1..fb91010d6ea1 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -71,6 +71,7 @@ from nova.openstack.common import importutils from nova.openstack.common import jsonutils from nova.openstack.common import log as logging from nova import utils +from nova.virt import configdrive from nova.virt.disk import api as disk from nova.virt import driver from nova.virt.libvirt import config @@ -1207,6 +1208,13 @@ class LibvirtDriver(driver.ComputeDriver): if not suffix: suffix = '' + # Are we using a config drive? + using_config_drive = False + if (instance.get('config_drive') or + FLAGS.force_config_drive): + LOG.info(_('Using config drive'), instance=instance) + using_config_drive = True + # syntactic nicety def basepath(fname='', suffix=suffix): return os.path.join(FLAGS.instances_path, @@ -1325,30 +1333,15 @@ class LibvirtDriver(driver.ComputeDriver): size=size, swap_mb=swap_mb) + # target partition for file injection target_partition = None if not instance['kernel_id']: target_partition = FLAGS.libvirt_inject_partition if target_partition == 0: target_partition = None - - config_drive, config_drive_id = self._get_config_drive_info(instance) - - if any((FLAGS.libvirt_type == 'lxc', config_drive, config_drive_id)): + if FLAGS.libvirt_type == 'lxc': target_partition = None - if config_drive_id: - fname = config_drive_id - raw('disk.config').cache(fn=libvirt_utils.fetch_image, - fname=fname, - image_id=config_drive_id, - user_id=instance['user_id'], - project_id=instance['project_id']) - elif config_drive: - label = 'config' - with utils.remove_path_on_error(basepath('disk.config')): - self._create_local(basepath('disk.config'), 64, unit='M', - fs_format='msdos', label=label) # 64MB - if FLAGS.libvirt_inject_key and instance['key_data']: key = str(instance['key_data']) else: @@ -1391,6 +1384,12 @@ class LibvirtDriver(driver.ComputeDriver): searchList=[{'interfaces': nets, 'use_ipv6': FLAGS.use_ipv6}])) + # Config drive + cdb = None + if using_config_drive: + cdb = configdrive.ConfigDriveBuilder(instance=instance) + + # File injection metadata = instance.get('metadata') if FLAGS.libvirt_inject_password: @@ -1401,28 +1400,45 @@ class LibvirtDriver(driver.ComputeDriver): files = instance.get('injected_files') if any((key, net, metadata, admin_pass, files)): - if config_drive: # Should be True or None by now. - injection_path = raw('disk.config').path - img_id = 'config-drive' - else: + if not using_config_drive: + # If we're not using config_drive, inject into root fs injection_path = image('disk').path img_id = instance['image_ref'] - for injection in ('metadata', 'key', 'net', 'admin_pass', 'files'): - if locals()[injection]: - LOG.info(_('Injecting %(injection)s into image' - ' %(img_id)s'), locals(), instance=instance) - try: - disk.inject_data(injection_path, - key, net, metadata, admin_pass, files, - partition=target_partition, - use_cow=FLAGS.use_cow_images) + for injection in ('metadata', 'key', 'net', 'admin_pass', + 'files'): + if locals()[injection]: + LOG.info(_('Injecting %(injection)s into image' + ' %(img_id)s'), locals(), instance=instance) + try: + disk.inject_data(injection_path, + key, net, metadata, admin_pass, files, + partition=target_partition, + use_cow=FLAGS.use_cow_images) - except Exception as e: - # This could be a windows image, or a vmdk format disk - LOG.warn(_('Ignoring error injecting data into image ' - '%(img_id)s (%(e)s)') % locals(), - instance=instance) + except Exception as e: + # This could be a windows image, or a vmdk format disk + LOG.warn(_('Ignoring error injecting data into image ' + '%(img_id)s (%(e)s)') % locals(), + instance=instance) + + else: + # We're using config_drive, so put the files there instead + cdb.inject_data(key, net, metadata, admin_pass, files) + + if using_config_drive: + # NOTE(mikal): Render the config drive. We can't add instance + # metadata here until after file injection, as the file injection + # creates state the openstack metadata relies on. + cdb.add_instance_metadata() + + try: + configdrive_path = basepath(fname='disk.config') + LOG.info(_('Creating config drive at %(path)s'), + {'path': configdrive_path}, instance=instance) + cdb.make_drive(configdrive_path) + finally: + cdb.cleanup() if FLAGS.libvirt_type == 'lxc': disk.setup_container(basepath('disk'), @@ -1450,18 +1466,6 @@ class LibvirtDriver(driver.ComputeDriver): LOG.debug(_("block_device_list %s"), block_device_list) return block_device.strip_dev(mount_device) in block_device_list - def _get_config_drive_info(self, instance): - config_drive = instance.get('config_drive') - config_drive_id = instance.get('config_drive_id') - if FLAGS.force_config_drive: - if not config_drive_id: - config_drive = True - return config_drive, config_drive_id - - def _has_config_drive(self, instance): - config_drive, config_drive_id = self._get_config_drive_info(instance) - return any((config_drive, config_drive_id)) - def get_host_capabilities(self): """Returns an instance of config.LibvirtConfigCaps representing the capabilities of the host""" @@ -1657,7 +1661,9 @@ class LibvirtDriver(driver.ComputeDriver): mount_device) devices.append(cfg) - if self._has_config_drive(instance): + if (instance.get('config_drive') or + instance.get('config_drive_id') or + FLAGS.force_config_drive): diskconfig = config.LibvirtConfigGuestDisk() diskconfig.source_type = "file" diskconfig.driver_format = "raw"