x/mogan
Code Issues Proposed changes

Add configdrive support - metadata

Browse Source 
This adds some simple metadata information, following up
patches will add more capabilities

Co-Authored-By: Shaohe Feng <shaohe.feng@intel.com>

Change-Id: Idaa3ab813b5355ce44e97fa069cd7b664f8b4761
This commit is contained in:
Zhenguo Niu 2017-04-07 15:23:36 +08:00
parent b3763ddd95
commit 8497848ac2
10 changed files with 517 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mogan/common/exception.py
View File

@ -376,4 +376,14 @@ class ConsoleTypeUnavailable(Invalid):
msg_fmt = _("Unavailable console type %(console_type)s.")
class ConfigDriveMountFailed(MoganException):
msg_fmt = _("Could not mount vfat config drive. %(operation)s failed. "
"Error: %(error)s")
class ConfigDriveUnknownFormat(MoganException):
msg_fmt = _("Unknown config drive format %(format)s. Select one of "
"iso9660 or vfat.")
ObjectActionError = obj_exc.ObjectActionError

74
mogan/common/utils.py
View File

@ -15,18 +15,24 @@
"""Utilities and helper functions."""
import contextlib
import functools
import inspect
import os
import re
import shutil
import tempfile
import traceback
from oslo_concurrency import lockutils
from oslo_concurrency import processutils
from oslo_log import log as logging
from oslo_utils import encodeutils
import six
from mogan.common import exception
from mogan.common import states
from mogan.conf import CONF
from mogan import objects
@ -209,3 +215,71 @@ def add_instance_fault_from_exc(context, instance, fault, exc_info=None,
code = fault_obj.code
fault_obj.detail = _get_fault_detail(exc_info, code)
fault_obj.create()
def execute(*cmd, **kwargs):
"""Convenience wrapper around oslo's execute() method.
:param cmd: Passed to processutils.execute.
:param use_standard_locale: True | False. Defaults to False. If set to
True, execute command with standard locale
added to environment variables.
:returns: (stdout, stderr) from process execution
:raises: UnknownArgumentError
:raises: ProcessExecutionError
"""
use_standard_locale = kwargs.pop('use_standard_locale', False)
if use_standard_locale:
env = kwargs.pop('env_variables', os.environ.copy())
env['LC_ALL'] = 'C'
kwargs['env_variables'] = env
result = processutils.execute(*cmd, **kwargs)
LOG.debug('Execution completed, command line is "%s"',
' '.join(map(str, cmd)))
LOG.debug('Command stdout is: "%s"', result[0])
LOG.debug('Command stderr is: "%s"', result[1])
return result
@contextlib.contextmanager
def tempdir(**kwargs):
tempfile.tempdir = CONF.tempdir
tmpdir = tempfile.mkdtemp(**kwargs)
try:
yield tmpdir
finally:
try:
shutil.rmtree(tmpdir)
except OSError as e:
LOG.error('Could not remove tmpdir: %s', e)
def mkfs(fs, path, label=None, run_as_root=False):
"""Format a file or block device
:param fs: Filesystem type (examples include 'swap', 'ext3', 'ext4'
'btrfs', etc.)
:param path: Path to file or block device to format
:param label: Volume label to use
"""
if fs == 'swap':
args = ['mkswap']
else:
args = ['mkfs', '-t', fs]
# add -F to force no interactive execute on non-block device.
if fs in ('ext3', 'ext4', 'ntfs'):
args.extend(['-F'])
if label:
if fs in ('msdos', 'vfat'):
label_opt = '-n'
else:
label_opt = '-L'
args.extend([label_opt, label])
args.append(path)
execute(*args, run_as_root=run_as_root)
def trycmd(*args, **kwargs):
"""Convenience wrapper around oslo's trycmd() method."""
return processutils.trycmd(*args, **kwargs)

2
mogan/conf/__init__.py
View File

@ -17,6 +17,7 @@ from oslo_config import cfg
from mogan.conf import api
from mogan.conf import cache
from mogan.conf import configdrive
from mogan.conf import database
from mogan.conf import default
from mogan.conf import engine
@ -31,6 +32,7 @@ from mogan.conf import shellinabox
CONF = cfg.CONF
api.register_opts(CONF)
configdrive.register_opts(CONF)
database.register_opts(CONF)
default.register_opts(CONF)
engine.register_opts(CONF)

34
mogan/conf/configdrive.py Normal file
View File

@ -0,0 +1,34 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from mogan.common.i18n import _
opts = [
cfg.StrOpt('config_drive_format',
default='iso9660',
choices=('iso9660', 'vfat'),
help=_('Configuration drive format that will contain '
'metadata attached to the instance when it boots.')),
cfg.StrOpt('mkisofs_cmd',
default='genisoimage',
help=_('Name or path of the tool used for ISO image '
'creation')),
]
def register_opts(conf):
conf.register_opts(opts, group='configdrive')

10
mogan/conf/default.py
View File

@ -15,6 +15,7 @@
import os
import socket
import tempfile
from oslo_config import cfg
@ -68,9 +69,18 @@ service_opts = [
),
]
utils_opts = [
cfg.StrOpt('tempdir',
default=tempfile.gettempdir(),
sample_default='/tmp',
help=_('Temporary working directory, default is Python temp '
'dir.')),
]
def register_opts(conf):
conf.register_opts(api_opts)
conf.register_opts(exc_log_opts)
conf.register_opts(service_opts)
conf.register_opts(path_opts)
conf.register_opts(utils_opts)

3
mogan/conf/opts.py
View File

@ -13,6 +13,7 @@
import itertools
import mogan.conf.api
import mogan.conf.configdrive
import mogan.conf.database
import mogan.conf.default
import mogan.conf.engine
@ -28,11 +29,13 @@ _default_opt_lists = [
mogan.conf.default.exc_log_opts,
mogan.conf.default.path_opts,
mogan.conf.default.service_opts,
mogan.conf.default.utils_opts,
]
_opts = [
('DEFAULT', itertools.chain(*_default_opt_lists)),
('api', mogan.conf.api.opts),
('configdrive', mogan.conf.configdrive.opts),
('database', mogan.conf.database.opts),
('engine', mogan.conf.engine.opts),
('glance', mogan.conf.glance.opts),

152
mogan/engine/baremetal/configdrive.py Normal file
View File

@ -0,0 +1,152 @@
# Copyright 2012 Michael Still and Canonical Inc
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Config Drive v2 helper."""
import os
import shutil
from oslo_utils import fileutils
from oslo_utils import units
import six
from mogan.common import exception
from mogan.common import utils
from mogan.conf import CONF
from mogan import version
# Config drives are 64mb, if we can't size to the exact size of the data
CONFIGDRIVESIZE_BYTES = 64 * units.Mi
class ConfigDriveBuilder(object):
"""Build config drives, optionally as a context manager."""
def __init__(self, instance_md=None):
self.imagefile = None
self.mdfiles = []
if instance_md is not None:
self.add_instance_metadata(instance_md)
def __enter__(self):
return self
def __exit__(self, exctype, excval, exctb):
if exctype is not None:
# NOTE(mikal): this means we're being cleaned up because an
# exception was thrown. All bets are off now, and we should not
# swallow the exception
return False
self.cleanup()
def _add_file(self, basedir, path, data):
filepath = os.path.join(basedir, path)
dirname = os.path.dirname(filepath)
fileutils.ensure_tree(dirname)
with open(filepath, 'wb') as f:
# the given data can be either text or bytes. we can only write
# bytes into files.
if isinstance(data, six.text_type):
data = data.encode('utf-8')
f.write(data)
def add_instance_metadata(self, instance_md):
for (path, data) in instance_md.metadata_for_config_drive():
self.mdfiles.append((path, data))
def _write_md_files(self, basedir):
for data in self.mdfiles:
self._add_file(basedir, data[0], data[1])
def _make_iso9660(self, path, tmpdir):
publisher = "%(product)s %(version)s" % {
'product': version.product_string(),
'version': version.version_string_with_package()}
utils.execute(CONF.configdrive.mkisofs_cmd,
'-o', path,
'-ldots',
'-allow-lowercase',
'-allow-multidot',
'-l',
'-publisher',
publisher,
'-quiet',
'-J',
'-r',
'-V', 'config-2',
tmpdir,
attempts=1,
run_as_root=False)
def _make_vfat(self, path, tmpdir):
# NOTE(mikal): This is a little horrible, but I couldn't find an
# equivalent to genisoimage for vfat filesystems.
with open(path, 'wb') as f:
f.truncate(CONFIGDRIVESIZE_BYTES)
utils.mkfs('vfat', path, label='config-2')
with utils.tempdir() as mountdir:
mounted = False
try:
_, err = utils.trycmd(
'mount', '-o', 'loop,uid=%d,gid=%d' % (os.getuid(),
os.getgid()),
path,
mountdir,
run_as_root=True)
if err:
raise exception.ConfigDriveMountFailed(operation='mount',
error=err)
mounted = True
# NOTE(mikal): I can't just use shutils.copytree here,
# because the destination directory already
# exists. This is annoying.
for ent in os.listdir(tmpdir):
shutil.copytree(os.path.join(tmpdir, ent),
os.path.join(mountdir, ent))
finally:
if mounted:
utils.execute('umount', mountdir, run_as_root=True)
def make_drive(self, path):
"""Make the config drive.
:param path: the path to place the config drive image at
:raises ProcessExecuteError if a helper process has failed.
"""
with utils.tempdir() as tmpdir:
self._write_md_files(tmpdir)
if CONF.configdrive.config_drive_format == 'iso9660':
self._make_iso9660(path, tmpdir)
elif CONF.configdrive.config_drive_format == 'vfat':
self._make_vfat(path, tmpdir)
else:
raise exception.ConfigDriveUnknownFormat(
format=CONF.configdrive.config_drive_format)
def cleanup(self):
if self.imagefile:
fileutils.delete_if_exists(self.imagefile)
def __repr__(self):
return "<ConfigDriveBuilder: " + str(self.mdfiles) + ">"

56
mogan/engine/baremetal/ironic/driver.py
View File

@ -13,6 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import base64
import gzip
import shutil
import tempfile
from ironicclient import exc as ironic_exc
from ironicclient import exceptions as client_e
from oslo_log import log as logging
@ -25,8 +30,10 @@ from mogan.common.i18n import _
from mogan.common import ironic
from mogan.common import states
from mogan.conf import CONF
from mogan.engine.baremetal import configdrive
from mogan.engine.baremetal import driver as base_driver
from mogan.engine.baremetal.ironic import ironic_states
from mogan import metadata as instance_metadata
LOG = logging.getLogger(__name__)
@ -283,6 +290,35 @@ class IronicDriver(base_driver.BaseEngineDriver):
except client_e.BadRequest:
pass
def _generate_configdrive(self, context, instance, node, extra_md=None):
"""Generate a config drive.
:param instance: The instance object.
:param node: The node object.
:param extra_md: Optional, extra metadata to be added to the
configdrive.
"""
if not extra_md:
extra_md = {}
i_meta = instance_metadata.InstanceMetadata(instance,
extra_md=extra_md)
with tempfile.NamedTemporaryFile() as uncompressed:
with configdrive.ConfigDriveBuilder(instance_md=i_meta) as cdb:
cdb.make_drive(uncompressed.name)
with tempfile.NamedTemporaryFile() as compressed:
# compress config drive
with gzip.GzipFile(fileobj=compressed, mode='wb') as gzipped:
uncompressed.seek(0)
shutil.copyfileobj(uncompressed, gzipped)
# base64 encode config drive
compressed.seek(0)
return base64.b64encode(compressed.read())
def spawn(self, context, instance):
"""Deploy an instance.
@ -316,10 +352,28 @@ class IronicDriver(base_driver.BaseEngineDriver):
'deploy': validate_chk.deploy,
'power': validate_chk.power})
# Config drive
configdrive_value = None
extra_md = {}
try:
configdrive_value = self._generate_configdrive(
context, instance, node, extra_md=extra_md)
except Exception as e:
with excutils.save_and_reraise_exception():
msg = ("Failed to build configdrive: %s" %
six.text_type(e))
LOG.error(msg, instance=instance)
LOG.info("Config drive for instance %(instance)s on "
"baremetal node %(node)s created.",
{'instance': instance['uuid'], 'node': node_uuid})
# trigger the node deploy
try:
self.ironicclient.call("node.set_provision_state", node_uuid,
ironic_states.ACTIVE)
ironic_states.ACTIVE,
configdrive=configdrive_value)
except Exception as e:
with excutils.save_and_reraise_exception():
msg = ("Failed to request Ironic to provision instance "

166
mogan/metadata.py Normal file
View File

@ -0,0 +1,166 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Instance Metadata information."""
import posixpath
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import timeutils
OCATA = '2017-02-22'
OPENSTACK_VERSIONS = [
OCATA,
]
VERSION = "version"
MD_JSON_NAME = "meta_data.json"
LOG = logging.getLogger(__name__)
class InvalidMetadataVersion(Exception):
pass
class InvalidMetadataPath(Exception):
pass
class InstanceMetadata(object):
"""Instance metadata."""
def __init__(self, instance, extra_md=None):
"""Creation of this object should basically cover all time consuming
collection. Methods after that should not cause time delays due to
network operations or lengthy cpu operations.
The user should then get a single instance and make multiple method
calls on it.
"""
self.instance = instance
self.extra_md = extra_md
self.availability_zone = instance.availability_zone
# TODO(zhenguo): Add hostname to instance object
self.hostname = instance.name
self.uuid = instance.uuid
self.files = []
self.route_configuration = None
def _route_configuration(self):
if self.route_configuration:
return self.route_configuration
path_handlers = {MD_JSON_NAME: self._metadata_as_json}
self.route_configuration = RouteConfiguration(path_handlers)
return self.route_configuration
def get_openstack_item(self, path_tokens):
return self._route_configuration().handle_path(path_tokens)
def _metadata_as_json(self, version, path):
metadata = {'uuid': self.uuid}
if self.extra_md:
metadata.update(self.extra_md)
metadata['hostname'] = self.hostname
metadata['name'] = self.instance.name
metadata['availability_zone'] = self.availability_zone
return jsonutils.dump_as_bytes(metadata)
def lookup(self, path):
if path == "" or path[0] != "/":
path = posixpath.normpath("/" + path)
else:
path = posixpath.normpath(path)
# fix up requests, prepending /openstack to anything that does
# not match
path_tokens = path.split('/')[1:]
if path_tokens[0] not in ("openstack"):
if path_tokens[0] == "":
# request for /
path_tokens = ["openstack"]
else:
path_tokens = ["openstack"] + path_tokens
path = "/" + "/".join(path_tokens)
# all values of 'path' input starts with '/' and have no trailing /
# specifically handle the top level request
if len(path_tokens) == 1:
if path_tokens[0] == "openstack":
# NOTE(vish): don't show versions that are in the future
today = timeutils.utcnow().strftime("%Y-%m-%d")
versions = [v for v in OPENSTACK_VERSIONS if v <= today]
if OPENSTACK_VERSIONS != versions:
LOG.debug("future versions %s hidden in version list",
[v for v in OPENSTACK_VERSIONS
if v not in versions], instance=self.instance)
versions += ["latest"]
return versions
try:
if path_tokens[0] == "openstack":
data = self.get_openstack_item(path_tokens[1:])
except (InvalidMetadataVersion, KeyError):
raise InvalidMetadataPath(path)
return data
def metadata_for_config_drive(self):
"""Yields (path, value) tuples for metadata elements."""
ALL_OPENSTACK_VERSIONS = OPENSTACK_VERSIONS + ["latest"]
for version in ALL_OPENSTACK_VERSIONS:
path = 'openstack/%s/%s' % (version, MD_JSON_NAME)
yield (path, self.lookup(path))
class RouteConfiguration(object):
"""Routes metadata paths to request handlers."""
def __init__(self, path_handler):
self.path_handlers = path_handler
def _version(self, version):
if version == "latest":
version = OPENSTACK_VERSIONS[-1]
if version not in OPENSTACK_VERSIONS:
raise InvalidMetadataVersion(version)
return version
def handle_path(self, path_tokens):
version = self._version(path_tokens[0])
if len(path_tokens) == 1:
path = VERSION
else:
path = '/'.join(path_tokens[1:])
path_handler = self.path_handlers[path]
if path_handler is None:
raise KeyError(path)
return path_handler(version, path)

11
mogan/version.py
View File

@ -15,4 +15,15 @@
import pbr.version
MOGAN_PRODUCT = "OpenStack Mogan"
version_info = pbr.version.VersionInfo('mogan')
version_string = version_info.version_string
def product_string():
return MOGAN_PRODUCT
def version_string_with_package():
return version_info.version_string()