diff --git a/mogan/common/exception.py b/mogan/common/exception.py index 704d8077..b2940ba4 100644 --- a/mogan/common/exception.py +++ b/mogan/common/exception.py @@ -376,4 +376,14 @@ class ConsoleTypeUnavailable(Invalid): msg_fmt = _("Unavailable console type %(console_type)s.") +class ConfigDriveMountFailed(MoganException): + msg_fmt = _("Could not mount vfat config drive. %(operation)s failed. " + "Error: %(error)s") + + +class ConfigDriveUnknownFormat(MoganException): + msg_fmt = _("Unknown config drive format %(format)s. Select one of " + "iso9660 or vfat.") + + ObjectActionError = obj_exc.ObjectActionError diff --git a/mogan/common/utils.py b/mogan/common/utils.py index 71bc5c6d..2285e876 100644 --- a/mogan/common/utils.py +++ b/mogan/common/utils.py @@ -15,18 +15,24 @@ """Utilities and helper functions.""" +import contextlib import functools import inspect +import os import re +import shutil +import tempfile import traceback from oslo_concurrency import lockutils +from oslo_concurrency import processutils from oslo_log import log as logging from oslo_utils import encodeutils import six from mogan.common import exception from mogan.common import states +from mogan.conf import CONF from mogan import objects @@ -209,3 +215,71 @@ def add_instance_fault_from_exc(context, instance, fault, exc_info=None, code = fault_obj.code fault_obj.detail = _get_fault_detail(exc_info, code) fault_obj.create() + + +def execute(*cmd, **kwargs): + """Convenience wrapper around oslo's execute() method. + + :param cmd: Passed to processutils.execute. + :param use_standard_locale: True | False. Defaults to False. If set to + True, execute command with standard locale + added to environment variables. + :returns: (stdout, stderr) from process execution + :raises: UnknownArgumentError + :raises: ProcessExecutionError + """ + + use_standard_locale = kwargs.pop('use_standard_locale', False) + if use_standard_locale: + env = kwargs.pop('env_variables', os.environ.copy()) + env['LC_ALL'] = 'C' + kwargs['env_variables'] = env + result = processutils.execute(*cmd, **kwargs) + LOG.debug('Execution completed, command line is "%s"', + ' '.join(map(str, cmd))) + LOG.debug('Command stdout is: "%s"', result[0]) + LOG.debug('Command stderr is: "%s"', result[1]) + return result + + +@contextlib.contextmanager +def tempdir(**kwargs): + tempfile.tempdir = CONF.tempdir + tmpdir = tempfile.mkdtemp(**kwargs) + try: + yield tmpdir + finally: + try: + shutil.rmtree(tmpdir) + except OSError as e: + LOG.error('Could not remove tmpdir: %s', e) + + +def mkfs(fs, path, label=None, run_as_root=False): + """Format a file or block device + + :param fs: Filesystem type (examples include 'swap', 'ext3', 'ext4' + 'btrfs', etc.) + :param path: Path to file or block device to format + :param label: Volume label to use + """ + if fs == 'swap': + args = ['mkswap'] + else: + args = ['mkfs', '-t', fs] + # add -F to force no interactive execute on non-block device. + if fs in ('ext3', 'ext4', 'ntfs'): + args.extend(['-F']) + if label: + if fs in ('msdos', 'vfat'): + label_opt = '-n' + else: + label_opt = '-L' + args.extend([label_opt, label]) + args.append(path) + execute(*args, run_as_root=run_as_root) + + +def trycmd(*args, **kwargs): + """Convenience wrapper around oslo's trycmd() method.""" + return processutils.trycmd(*args, **kwargs) diff --git a/mogan/conf/__init__.py b/mogan/conf/__init__.py index 8885d9ef..e88a9e3b 100644 --- a/mogan/conf/__init__.py +++ b/mogan/conf/__init__.py @@ -17,6 +17,7 @@ from oslo_config import cfg from mogan.conf import api from mogan.conf import cache +from mogan.conf import configdrive from mogan.conf import database from mogan.conf import default from mogan.conf import engine @@ -31,6 +32,7 @@ from mogan.conf import shellinabox CONF = cfg.CONF api.register_opts(CONF) +configdrive.register_opts(CONF) database.register_opts(CONF) default.register_opts(CONF) engine.register_opts(CONF) diff --git a/mogan/conf/configdrive.py b/mogan/conf/configdrive.py new file mode 100644 index 00000000..56f6fcfa --- /dev/null +++ b/mogan/conf/configdrive.py @@ -0,0 +1,34 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from mogan.common.i18n import _ + +opts = [ + cfg.StrOpt('config_drive_format', + default='iso9660', + choices=('iso9660', 'vfat'), + help=_('Configuration drive format that will contain ' + 'metadata attached to the instance when it boots.')), + cfg.StrOpt('mkisofs_cmd', + default='genisoimage', + help=_('Name or path of the tool used for ISO image ' + 'creation')), +] + + +def register_opts(conf): + conf.register_opts(opts, group='configdrive') diff --git a/mogan/conf/default.py b/mogan/conf/default.py index 31d1df13..ef3d5ebc 100644 --- a/mogan/conf/default.py +++ b/mogan/conf/default.py @@ -15,6 +15,7 @@ import os import socket +import tempfile from oslo_config import cfg @@ -68,9 +69,18 @@ service_opts = [ ), ] +utils_opts = [ + cfg.StrOpt('tempdir', + default=tempfile.gettempdir(), + sample_default='/tmp', + help=_('Temporary working directory, default is Python temp ' + 'dir.')), +] + def register_opts(conf): conf.register_opts(api_opts) conf.register_opts(exc_log_opts) conf.register_opts(service_opts) conf.register_opts(path_opts) + conf.register_opts(utils_opts) diff --git a/mogan/conf/opts.py b/mogan/conf/opts.py index a25cfccf..08991191 100644 --- a/mogan/conf/opts.py +++ b/mogan/conf/opts.py @@ -13,6 +13,7 @@ import itertools import mogan.conf.api +import mogan.conf.configdrive import mogan.conf.database import mogan.conf.default import mogan.conf.engine @@ -28,11 +29,13 @@ _default_opt_lists = [ mogan.conf.default.exc_log_opts, mogan.conf.default.path_opts, mogan.conf.default.service_opts, + mogan.conf.default.utils_opts, ] _opts = [ ('DEFAULT', itertools.chain(*_default_opt_lists)), ('api', mogan.conf.api.opts), + ('configdrive', mogan.conf.configdrive.opts), ('database', mogan.conf.database.opts), ('engine', mogan.conf.engine.opts), ('glance', mogan.conf.glance.opts), diff --git a/mogan/engine/baremetal/configdrive.py b/mogan/engine/baremetal/configdrive.py new file mode 100644 index 00000000..c2526021 --- /dev/null +++ b/mogan/engine/baremetal/configdrive.py @@ -0,0 +1,152 @@ +# Copyright 2012 Michael Still and Canonical Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Config Drive v2 helper.""" + +import os +import shutil + +from oslo_utils import fileutils +from oslo_utils import units +import six + +from mogan.common import exception +from mogan.common import utils +from mogan.conf import CONF +from mogan import version + + +# Config drives are 64mb, if we can't size to the exact size of the data +CONFIGDRIVESIZE_BYTES = 64 * units.Mi + + +class ConfigDriveBuilder(object): + """Build config drives, optionally as a context manager.""" + + def __init__(self, instance_md=None): + self.imagefile = None + self.mdfiles = [] + + if instance_md is not None: + self.add_instance_metadata(instance_md) + + def __enter__(self): + return self + + def __exit__(self, exctype, excval, exctb): + if exctype is not None: + # NOTE(mikal): this means we're being cleaned up because an + # exception was thrown. All bets are off now, and we should not + # swallow the exception + return False + self.cleanup() + + def _add_file(self, basedir, path, data): + filepath = os.path.join(basedir, path) + dirname = os.path.dirname(filepath) + fileutils.ensure_tree(dirname) + with open(filepath, 'wb') as f: + # the given data can be either text or bytes. we can only write + # bytes into files. + if isinstance(data, six.text_type): + data = data.encode('utf-8') + f.write(data) + + def add_instance_metadata(self, instance_md): + for (path, data) in instance_md.metadata_for_config_drive(): + self.mdfiles.append((path, data)) + + def _write_md_files(self, basedir): + for data in self.mdfiles: + self._add_file(basedir, data[0], data[1]) + + def _make_iso9660(self, path, tmpdir): + publisher = "%(product)s %(version)s" % { + 'product': version.product_string(), + 'version': version.version_string_with_package()} + + utils.execute(CONF.configdrive.mkisofs_cmd, + '-o', path, + '-ldots', + '-allow-lowercase', + '-allow-multidot', + '-l', + '-publisher', + publisher, + '-quiet', + '-J', + '-r', + '-V', 'config-2', + tmpdir, + attempts=1, + run_as_root=False) + + def _make_vfat(self, path, tmpdir): + # NOTE(mikal): This is a little horrible, but I couldn't find an + # equivalent to genisoimage for vfat filesystems. + with open(path, 'wb') as f: + f.truncate(CONFIGDRIVESIZE_BYTES) + + utils.mkfs('vfat', path, label='config-2') + + with utils.tempdir() as mountdir: + mounted = False + try: + _, err = utils.trycmd( + 'mount', '-o', 'loop,uid=%d,gid=%d' % (os.getuid(), + os.getgid()), + path, + mountdir, + run_as_root=True) + if err: + raise exception.ConfigDriveMountFailed(operation='mount', + error=err) + mounted = True + + # NOTE(mikal): I can't just use shutils.copytree here, + # because the destination directory already + # exists. This is annoying. + for ent in os.listdir(tmpdir): + shutil.copytree(os.path.join(tmpdir, ent), + os.path.join(mountdir, ent)) + + finally: + if mounted: + utils.execute('umount', mountdir, run_as_root=True) + + def make_drive(self, path): + """Make the config drive. + + :param path: the path to place the config drive image at + + :raises ProcessExecuteError if a helper process has failed. + """ + with utils.tempdir() as tmpdir: + self._write_md_files(tmpdir) + + if CONF.configdrive.config_drive_format == 'iso9660': + self._make_iso9660(path, tmpdir) + elif CONF.configdrive.config_drive_format == 'vfat': + self._make_vfat(path, tmpdir) + else: + raise exception.ConfigDriveUnknownFormat( + format=CONF.configdrive.config_drive_format) + + def cleanup(self): + if self.imagefile: + fileutils.delete_if_exists(self.imagefile) + + def __repr__(self): + return "" diff --git a/mogan/engine/baremetal/ironic/driver.py b/mogan/engine/baremetal/ironic/driver.py index 270286e3..1826be20 100644 --- a/mogan/engine/baremetal/ironic/driver.py +++ b/mogan/engine/baremetal/ironic/driver.py @@ -13,6 +13,11 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 +import gzip +import shutil +import tempfile + from ironicclient import exc as ironic_exc from ironicclient import exceptions as client_e from oslo_log import log as logging @@ -25,8 +30,10 @@ from mogan.common.i18n import _ from mogan.common import ironic from mogan.common import states from mogan.conf import CONF +from mogan.engine.baremetal import configdrive from mogan.engine.baremetal import driver as base_driver from mogan.engine.baremetal.ironic import ironic_states +from mogan import metadata as instance_metadata LOG = logging.getLogger(__name__) @@ -283,6 +290,35 @@ class IronicDriver(base_driver.BaseEngineDriver): except client_e.BadRequest: pass + def _generate_configdrive(self, context, instance, node, extra_md=None): + """Generate a config drive. + + :param instance: The instance object. + :param node: The node object. + :param extra_md: Optional, extra metadata to be added to the + configdrive. + + """ + if not extra_md: + extra_md = {} + + i_meta = instance_metadata.InstanceMetadata(instance, + extra_md=extra_md) + + with tempfile.NamedTemporaryFile() as uncompressed: + with configdrive.ConfigDriveBuilder(instance_md=i_meta) as cdb: + cdb.make_drive(uncompressed.name) + + with tempfile.NamedTemporaryFile() as compressed: + # compress config drive + with gzip.GzipFile(fileobj=compressed, mode='wb') as gzipped: + uncompressed.seek(0) + shutil.copyfileobj(uncompressed, gzipped) + + # base64 encode config drive + compressed.seek(0) + return base64.b64encode(compressed.read()) + def spawn(self, context, instance): """Deploy an instance. @@ -316,10 +352,28 @@ class IronicDriver(base_driver.BaseEngineDriver): 'deploy': validate_chk.deploy, 'power': validate_chk.power}) + # Config drive + configdrive_value = None + extra_md = {} + + try: + configdrive_value = self._generate_configdrive( + context, instance, node, extra_md=extra_md) + except Exception as e: + with excutils.save_and_reraise_exception(): + msg = ("Failed to build configdrive: %s" % + six.text_type(e)) + LOG.error(msg, instance=instance) + + LOG.info("Config drive for instance %(instance)s on " + "baremetal node %(node)s created.", + {'instance': instance['uuid'], 'node': node_uuid}) + # trigger the node deploy try: self.ironicclient.call("node.set_provision_state", node_uuid, - ironic_states.ACTIVE) + ironic_states.ACTIVE, + configdrive=configdrive_value) except Exception as e: with excutils.save_and_reraise_exception(): msg = ("Failed to request Ironic to provision instance " diff --git a/mogan/metadata.py b/mogan/metadata.py new file mode 100644 index 00000000..09433338 --- /dev/null +++ b/mogan/metadata.py @@ -0,0 +1,166 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Instance Metadata information.""" + +import posixpath + +from oslo_log import log as logging +from oslo_serialization import jsonutils +from oslo_utils import timeutils + + +OCATA = '2017-02-22' + +OPENSTACK_VERSIONS = [ + OCATA, +] + +VERSION = "version" +MD_JSON_NAME = "meta_data.json" + +LOG = logging.getLogger(__name__) + + +class InvalidMetadataVersion(Exception): + pass + + +class InvalidMetadataPath(Exception): + pass + + +class InstanceMetadata(object): + """Instance metadata.""" + + def __init__(self, instance, extra_md=None): + """Creation of this object should basically cover all time consuming + collection. Methods after that should not cause time delays due to + network operations or lengthy cpu operations. + + The user should then get a single instance and make multiple method + calls on it. + """ + + self.instance = instance + self.extra_md = extra_md + self.availability_zone = instance.availability_zone + # TODO(zhenguo): Add hostname to instance object + self.hostname = instance.name + self.uuid = instance.uuid + self.files = [] + + self.route_configuration = None + + def _route_configuration(self): + if self.route_configuration: + return self.route_configuration + + path_handlers = {MD_JSON_NAME: self._metadata_as_json} + + self.route_configuration = RouteConfiguration(path_handlers) + return self.route_configuration + + def get_openstack_item(self, path_tokens): + return self._route_configuration().handle_path(path_tokens) + + def _metadata_as_json(self, version, path): + metadata = {'uuid': self.uuid} + if self.extra_md: + metadata.update(self.extra_md) + + metadata['hostname'] = self.hostname + metadata['name'] = self.instance.name + metadata['availability_zone'] = self.availability_zone + + return jsonutils.dump_as_bytes(metadata) + + def lookup(self, path): + if path == "" or path[0] != "/": + path = posixpath.normpath("/" + path) + else: + path = posixpath.normpath(path) + + # fix up requests, prepending /openstack to anything that does + # not match + path_tokens = path.split('/')[1:] + if path_tokens[0] not in ("openstack"): + if path_tokens[0] == "": + # request for / + path_tokens = ["openstack"] + else: + path_tokens = ["openstack"] + path_tokens + path = "/" + "/".join(path_tokens) + + # all values of 'path' input starts with '/' and have no trailing / + + # specifically handle the top level request + if len(path_tokens) == 1: + if path_tokens[0] == "openstack": + # NOTE(vish): don't show versions that are in the future + today = timeutils.utcnow().strftime("%Y-%m-%d") + versions = [v for v in OPENSTACK_VERSIONS if v <= today] + if OPENSTACK_VERSIONS != versions: + LOG.debug("future versions %s hidden in version list", + [v for v in OPENSTACK_VERSIONS + if v not in versions], instance=self.instance) + versions += ["latest"] + return versions + + try: + if path_tokens[0] == "openstack": + data = self.get_openstack_item(path_tokens[1:]) + except (InvalidMetadataVersion, KeyError): + raise InvalidMetadataPath(path) + + return data + + def metadata_for_config_drive(self): + """Yields (path, value) tuples for metadata elements.""" + ALL_OPENSTACK_VERSIONS = OPENSTACK_VERSIONS + ["latest"] + for version in ALL_OPENSTACK_VERSIONS: + path = 'openstack/%s/%s' % (version, MD_JSON_NAME) + yield (path, self.lookup(path)) + + +class RouteConfiguration(object): + """Routes metadata paths to request handlers.""" + + def __init__(self, path_handler): + self.path_handlers = path_handler + + def _version(self, version): + if version == "latest": + version = OPENSTACK_VERSIONS[-1] + + if version not in OPENSTACK_VERSIONS: + raise InvalidMetadataVersion(version) + + return version + + def handle_path(self, path_tokens): + version = self._version(path_tokens[0]) + if len(path_tokens) == 1: + path = VERSION + else: + path = '/'.join(path_tokens[1:]) + + path_handler = self.path_handlers[path] + + if path_handler is None: + raise KeyError(path) + + return path_handler(version, path) diff --git a/mogan/version.py b/mogan/version.py index 621bfb0c..74ac1466 100644 --- a/mogan/version.py +++ b/mogan/version.py @@ -15,4 +15,15 @@ import pbr.version +MOGAN_PRODUCT = "OpenStack Mogan" + version_info = pbr.version.VersionInfo('mogan') +version_string = version_info.version_string + + +def product_string(): + return MOGAN_PRODUCT + + +def version_string_with_package(): + return version_info.version_string()