Added Glance Client for Image downloading

According to sean mooney's comments in
https://review.openstack.org/#/c/596507/

I am splitting into 2 patches

For now, please add following session in cyborg.conf
[glance]
project_domain_name = Default
project_name = service
user_domain_name = Default
password = {your_password}
username = placement
auth_url = http://{host_ip}/identity
auth_type = password
auth_section = keystone_authtoken
api_servers = http://{host_ip}/image

Change-Id: I3a0249c7ac8ba5fd3c20ae6402dcc2f9e9bcfe95
This commit is contained in:
Li Liu 2018-10-11 23:25:06 -04:00
parent 715f03953a
commit 156c9f18e5
14 changed files with 1498 additions and 3 deletions

View File

@ -305,3 +305,25 @@ class QuotaResourceUnknown(QuotaNotFound):
class InvalidReservationExpiration(Invalid):
message = _("Invalid reservation expiration %(expire)s.")
class GlanceConnectionFailed(CyborgException):
msg_fmt = _("Connection to glance host %(server)s failed: "
"%(reason)s")
class ImageUnacceptable(Invalid):
msg_fmt = _("Image %(image_id)s is unacceptable: %(reason)s")
class ImageNotAuthorized(CyborgException):
msg_fmt = _("Not authorized for image %(image_id)s.")
class ImageNotFound(NotFound):
msg_fmt = _("Image %(image_id)s could not be found.")
class ImageBadRequest(Invalid):
msg_fmt = _("Request of image %(image_id)s got BadRequest response: "
"%(response)s")

View File

@ -14,7 +14,7 @@
# under the License.
from oslo_config import cfg
from oslo_context import context as cyborg_context
from cyborg import context as cyborg_context
import oslo_messaging as messaging
from oslo_messaging.rpc import dispatcher

View File

@ -14,7 +14,7 @@
# under the License.
from oslo_concurrency import processutils
from oslo_context import context
from cyborg import context
from oslo_log import log
import oslo_messaging as messaging
from oslo_service import service

View File

@ -108,3 +108,57 @@ def get_ksa_adapter(service_type, ksa_auth=None, ksa_session=None,
return ks_loading.load_adapter_from_conf_options(
CONF, confgrp, session=ksa_session, auth=ksa_auth,
min_version=min_version, max_version=max_version)
def get_endpoint(ksa_adapter):
"""Get the endpoint URL represented by a keystoneauth1 Adapter.
This method is equivalent to what
ksa_adapter.get_endpoint()
should do, if it weren't for a panoply of bugs.
:param ksa_adapter: keystoneauth1.adapter.Adapter, appropriately set up
with an endpoint_override; or service_type, interface
(list) and auth/service_catalog.
:return: String endpoint URL.
:raise EndpointNotFound: If endpoint discovery fails.
"""
# TODO(efried): This will be unnecessary once bug #1707993 is fixed.
# (At least for the non-image case, until 1707995 is fixed.)
if ksa_adapter.endpoint_override:
return ksa_adapter.endpoint_override
# TODO(efried): Remove this once bug #1707995 is fixed.
if ksa_adapter.service_type == 'image':
try:
# LOG.warning(ksa_adapter.__dict__)
return ksa_adapter.get_endpoint_data().catalog_url
except AttributeError:
# ksa_adapter.auth is a _ContextAuthPlugin, which doesn't have
# get_endpoint_data. Fall through to using get_endpoint().
pass
# TODO(efried): The remainder of this method reduces to
# TODO(efried): return ksa_adapter.get_endpoint()
# TODO(efried): once bug #1709118 is fixed.
# NOTE(efried): Id9bd19cca68206fc64d23b0eaa95aa3e5b01b676 may also do the
# trick, once it's in a ksa release.
# The EndpointNotFound exception happens when _ContextAuthPlugin is in play
# because its get_endpoint() method isn't yet set up to handle interface as
# a list. (It could also happen with a real auth if the endpoint isn't
# there; but that's covered below.)
try:
return ksa_adapter.get_endpoint()
except ks_exc.EndpointNotFound:
pass
interfaces = list(ksa_adapter.interface)
for interface in interfaces:
ksa_adapter.interface = interface
try:
return ksa_adapter.get_endpoint()
except ks_exc.EndpointNotFound:
pass
raise ks_exc.EndpointNotFound(
"Could not find requested endpoint for any of the following "
"interfaces: %s" % interfaces)

View File

@ -18,7 +18,9 @@ from oslo_config import cfg
from cyborg.conf import api
from cyborg.conf import database
from cyborg.conf import default
from cyborg.conf import service_token
from cyborg.conf import glance
from cyborg.conf import keystone
CONF = cfg.CONF
@ -26,3 +28,6 @@ api.register_opts(CONF)
database.register_opts(CONF)
default.register_opts(CONF)
default.register_placement_opts(CONF)
service_token.register_opts(CONF)
glance.register_opts(CONF)
keystone.register_opts(CONF)

171
cyborg/conf/glance.py Normal file
View File

@ -0,0 +1,171 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from cyborg.conf import utils as confutils
DEFAULT_SERVICE_TYPE = 'image'
glance_group = cfg.OptGroup(
'glance',
title='Glance Options',
help='Configuration options for the Image service')
glance_opts = [
# NOTE(sdague/efried): there is intentionally no default here. This
# requires configuration if ksa adapter config is not used.
cfg.ListOpt('api_servers',
help="""
List of glance api servers endpoints available to cyborg.
https is used for ssl-based glance api servers.
NOTE: The preferred mechanism for endpoint discovery is via keystoneauth1
loading options. Only use api_servers if you need multiple endpoints and are
unable to use a load balancer for some reason.
Possible values:
* A list of any fully qualified url of the form "scheme://hostname:port[/path]"
(i.e. "http://10.0.1.0:9292" or "https://my.glance.server/image").
"""),
cfg.IntOpt('num_retries',
default=0,
min=0,
help="""
Enable glance operation retries.
Specifies the number of retries when uploading / downloading
an image to / from glance. 0 means no retries.
"""),
cfg.ListOpt('allowed_direct_url_schemes',
default=[],
deprecated_for_removal=True,
deprecated_since='17.0.0',
deprecated_reason="""
This was originally added for the 'cyborg.image.download.file' FileTransfer
extension which was removed in the 16.0.0 Pike release. The
'cyborg.image.download.modules' extension point is not maintained
and there is no indication of its use in production clouds.
""",
help="""
List of url schemes that can be directly accessed.
This option specifies a list of url schemes that can be downloaded
directly via the direct_url. This direct_URL can be fetched from
Image metadata which can be used by cyborg to get the
image more efficiently. cyborg-compute could benefit from this by
invoking a copy when it has access to the same file system as glance.
Possible values:
* [file], Empty list (default)
"""),
cfg.BoolOpt('verify_glance_signatures',
default=False,
help="""
Enable image signature verification.
cyborg uses the image signature metadata from glance and verifies the signature
of a signed image while downloading that image. If the image signature cannot
be verified or if the image signature metadata is either incomplete or
unavailable, then cyborg will not boot the image and instead will place the
instance into an error state. This provides end users with stronger assurances
of the integrity of the image data they are using to create servers.
Related options:
* The options in the `key_manager` group, as the key_manager is used
for the signature validation.
* Both enable_certificate_validation and default_trusted_certificate_ids
below depend on this option being enabled.
"""),
cfg.BoolOpt('enable_certificate_validation',
default=False,
deprecated_for_removal=True,
deprecated_since='16.0.0',
deprecated_reason="""
This option is intended to ease the transition for deployments leveraging
image signature verification. The intended state long-term is for signature
verification and certificate validation to always happen together.
""",
help="""
Enable certificate validation for image signature verification.
During image signature verification cyborg will first verify the validity of
the image's signing certificate using the set of trusted certificates
associated with the instance. If certificate validation fails, signature
verification will not be performed and the instance will be placed into an
error state. This provides end users with stronger assurances that the image
data is unmodified and trustworthy. If left disabled, image signature
verification can still occur but the end user will not have any assurance that
the signing certificate used to generate the image signature is still
trustworthy.
Related options:
* This option only takes effect if verify_glance_signatures is enabled.
* The value of default_trusted_certificate_ids may be used when this option
is enabled.
"""),
cfg.ListOpt('default_trusted_certificate_ids',
default=[],
help="""
List of certificate IDs for certificates that should be trusted.
May be used as a default list of trusted certificate IDs for certificate
validation. The value of this option will be ignored if the user provides a
list of trusted certificate IDs with an instance API request. The value of
this option will be persisted with the instance data if signature verification
and certificate validation are enabled and if the user did not provide an
alternative list. If left empty when certificate validation is enabled the
user must provide a list of trusted certificate IDs otherwise certificate
validation will fail.
Related options:
* The value of this option may be used if both verify_glance_signatures and
enable_certificate_validation are enabled.
"""),
cfg.BoolOpt('debug',
default=False,
help='Enable or disable debug logging with glanceclient.')
]
deprecated_ksa_opts = {
'insecure': [cfg.DeprecatedOpt('api_insecure', group=glance_group.name)],
'cafile': [cfg.DeprecatedOpt('ca_file', group="ssl")],
'certfile': [cfg.DeprecatedOpt('cert_file', group="ssl")],
'keyfile': [cfg.DeprecatedOpt('key_file', group="ssl")],
}
def register_opts(conf):
conf.register_group(glance_group)
conf.register_opts(glance_opts, group=glance_group)
confutils.register_ksa_opts(
conf, glance_group, DEFAULT_SERVICE_TYPE, include_auth=False,
deprecated_opts=deprecated_ksa_opts)
def list_opts():
return {glance_group: (
glance_opts +
ks_loading.get_session_conf_options() +
confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE,
deprecated_opts=deprecated_ksa_opts))}

40
cyborg/conf/keystone.py Normal file
View File

@ -0,0 +1,40 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from cyborg.conf import utils as confutils
DEFAULT_SERVICE_TYPE = 'identity'
keystone_group = cfg.OptGroup(
'keystone',
title='Keystone Options',
help='Configuration options for the identity service')
def register_opts(conf):
conf.register_group(keystone_group)
confutils.register_ksa_opts(conf, keystone_group.name,
DEFAULT_SERVICE_TYPE, include_auth=False)
def list_opts():
return {
keystone_group: (
ks_loading.get_session_conf_options() +
confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE)
)
}

View File

@ -0,0 +1,54 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
SERVICE_USER_GROUP = 'service_user'
service_user = cfg.OptGroup(
SERVICE_USER_GROUP,
title='Service token authentication type options',
help="""
Configuration options for service to service authentication using a service
token. These options allow sending a service token along with the user's token
when contacting external REST APIs.
"""
)
service_user_opts = [
cfg.BoolOpt('send_service_user_token',
default=False,
help="""
When True, if sending a user token to a REST API, also send a service token.
"""),
]
def register_opts(conf):
conf.register_group(service_user)
conf.register_opts(service_user_opts, group=service_user)
ks_loading.register_session_conf_options(conf, SERVICE_USER_GROUP)
ks_loading.register_auth_conf_options(conf, SERVICE_USER_GROUP)
def list_opts():
return {
service_user: (
service_user_opts +
ks_loading.get_session_conf_options() +
ks_loading.get_auth_common_conf_options() +
ks_loading.get_auth_plugin_conf_options('password') +
ks_loading.get_auth_plugin_conf_options('v2password') +
ks_loading.get_auth_plugin_conf_options('v3password'))
}

171
cyborg/context.py Normal file
View File

@ -0,0 +1,171 @@
# Copyright 2011 OpenStack Foundation
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystoneauth1.access import service_catalog as ksa_service_catalog
from keystoneauth1 import plugin
from oslo_context import context
from oslo_db.sqlalchemy import enginefacade
from oslo_utils import timeutils
import six
class _ContextAuthPlugin(plugin.BaseAuthPlugin):
"""A keystoneauth auth plugin that uses the values from the Context.
Ideally we would use the plugin provided by auth_token middleware however
this plugin isn't serialized yet so we construct one from the serialized
auth data.
"""
def __init__(self, auth_token, sc):
super(_ContextAuthPlugin, self).__init__()
self.auth_token = auth_token
self.service_catalog = ksa_service_catalog.ServiceCatalogV2(sc)
def get_token(self, *args, **kwargs):
return self.auth_token
def get_endpoint(self, session, service_type=None, interface=None,
region_name=None, service_name=None, **kwargs):
return self.service_catalog.url_for(service_type=service_type,
service_name=service_name,
interface=interface,
region_name=region_name)
@enginefacade.transaction_context_provider
class RequestContext(context.RequestContext):
"""Security context and request information.
Represents the user taking a given action within the system.
"""
def __init__(self, user_id=None, project_id=None, is_admin=None,
read_deleted="no", remote_address=None, timestamp=None,
quota_class=None, service_catalog=None,
user_auth_plugin=None, **kwargs):
""":param read_deleted: 'no' indicates deleted records are hidden,
'yes' indicates deleted records are visible,
'only' indicates that *only* deleted records are visible.
:param overwrite: Set to False to ensure that the greenthread local
copy of the index is not overwritten.
:param instance_lock_checked: This is not used and will be removed
in a future release.
:param user_auth_plugin: The auth plugin for the current request's
authentication data.
"""
if user_id:
kwargs['user_id'] = user_id
if project_id:
kwargs['project_id'] = project_id
super(RequestContext, self).__init__(is_admin=is_admin, **kwargs)
self.read_deleted = read_deleted
self.remote_address = remote_address
if not timestamp:
timestamp = timeutils.utcnow()
if isinstance(timestamp, six.string_types):
timestamp = timeutils.parse_strtime(timestamp)
self.timestamp = timestamp
if service_catalog:
# Only include required parts of service_catalog
self.service_catalog = [s for s in service_catalog
if s.get('type') in ('image')]
else:
# if list is empty or none
self.service_catalog = []
self.user_auth_plugin = user_auth_plugin
# if self.is_admin is None:
# self.is_admin = policy.check_is_admin(self)
def get_auth_plugin(self):
if self.user_auth_plugin:
return self.user_auth_plugin
else:
return _ContextAuthPlugin(self.auth_token, self.service_catalog)
def get_context():
"""A helper method to get a blank context.
Note that overwrite is False here so this context will not update the
greenthread-local stored context that is used when logging.
"""
return RequestContext(user_id=None,
project_id=None,
is_admin=False,
overwrite=False)
def get_admin_context(read_deleted="no"):
# NOTE(alaski): This method should only be used when an admin context is
# necessary for the entirety of the context lifetime. If that's not the
# case please use get_context(), or create the RequestContext manually, and
# use context.elevated() where necessary. Some periodic tasks may use
# get_admin_context so that their database calls are not filtered on
# project_id.
return RequestContext(user_id=None,
project_id=None,
is_admin=True,
read_deleted=read_deleted,
overwrite=False)
def is_user_context(context):
"""Indicates if the request context is a normal user."""
if not context:
return False
if context.is_admin:
return False
if not context.user_id or not context.project_id:
return False
return True
def require_context(ctxt):
"""Raise exception.Forbidden() if context is not a user or an
admin context.
"""
if not ctxt.is_admin and not is_user_context(ctxt):
raise exception.Forbidden()
def authorize_project_context(context, project_id):
"""Ensures a request has permission to access the given project."""
if is_user_context(context):
if not context.project_id:
raise exception.Forbidden()
elif context.project_id != project_id:
raise exception.Forbidden()
def authorize_user_context(context, user_id):
"""Ensures a request has permission to access the given user."""
if is_user_context(context):
if not context.user_id:
raise exception.Forbidden()
elif context.user_id != user_id:
raise exception.Forbidden()

0
cyborg/image/__init__.py Normal file
View File

163
cyborg/image/api.py Normal file
View File

@ -0,0 +1,163 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Main abstraction layer for retrieving and storing information about acclerator
images used by the cyborg agent layer.
"""
from cyborg.image import glance
from oslo_log import log
LOG = log.getLogger(__name__)
class API(object):
"""Responsible for exposing a relatively stable internal API for other
modules in Cyborg to retrieve information about acclerator images.
"""
def _get_session_and_image_id(self, context, id_or_uri):
"""Returns a tuple of (session, image_id). If the supplied `id_or_uri`
is an image ID, then the default client session will be returned
for the context's user, along with the image ID. If the supplied
`id_or_uri` parameter is a URI, then a client session connecting to
the URI's image service endpoint will be returned along with a
parsed image ID from that URI.
:param context: The `cyborg.context.Context` object for the request
:param id_or_uri: A UUID identifier or an image URI to look up image
information for.
"""
return glance.get_remote_image_service(context, id_or_uri)
def _get_session(self, _context):
"""Returns a client session that can be used to query for image
information.
:param _context: The `cyborg.context.Context` object for the request
"""
return glance.get_default_image_service()
@staticmethod
def generate_image_url(image_ref, context):
"""Generate an image URL from an image_ref.
:param image_ref: The image ref to generate URL
:param context: The `cyborg.context.Context` object for the request
"""
return "%s/images/%s" % (next(glance.get_api_servers(context)),
image_ref)
def get_all(self, context, **kwargs):
"""Retrieves all information records about all acclerator images
available to show to the requesting user. If the requesting user is an
admin, all images in an ACTIVE status are returned. If the requesting
user is not an admin, the all public images and all private images
that are owned by the requesting user in the ACTIVE status are
returned.
:param context: The `cyborg.context.Context` object for the request
:param kwargs: A dictionary of filter and pagination values that
may be passed to the underlying image info driver.
"""
session = self._get_session(context)
return session.detail(context, **kwargs)
def get(self, context, id_or_uri):
"""Retrieves the information record for a single acclerator image.
If the supplied identifier parameter is a UUID, the default driver will
be used to return information about the image. If the supplied
identifier is a URI, then the driver that matches that URI endpoint
will be used to query for image information.
:param context: The `cyborg.context.Context` object for the request
:param id_or_uri: A UUID identifier or an image URI to look up image
information for.
"""
session, image_id = self._get_session_and_image_id(context, id_or_uri)
return session.show(context, image_id,
include_locations=False,
show_deleted=False)
def update(self, context, id_or_uri, image_info,
data=None, purge_props=False):
"""Update the information about an image, optionally along with a file
handle or bytestream iterator for image bits. If the optional file
handle for updated image bits is supplied, the image may not have
already uploaded bits for the image.
:param context: The `cyborg.context.Context` object for the request
:param id_or_uri: A UUID identifier or an image URI to look up image
information for.
:param image_info: A dict of information about the image that is
passed to the image registry.
:param data: Optional file handle or bytestream iterator that is
passed to backend storage.
:param purge_props: Optional, defaults to False. If set, the backend
image registry will clear all image properties
and replace them the image properties supplied
in the image_info dictionary's 'properties'
collection.
"""
session, image_id = self._get_session_and_image_id(context, id_or_uri)
return session.update(context, image_id, image_info, data=data,
purge_props=purge_props)
def delete(self, context, id_or_uri):
"""Delete the information about an image and mark the image bits for
deletion.
:param context: The `cyborg.context.Context` object for the request
:param id_or_uri: A UUID identifier or an image URI to look up image
information for.
"""
session, image_id = self._get_session_and_image_id(context, id_or_uri)
return session.delete(context, image_id)
def download(self, context, id_or_uri, data=None, dest_path=None):
"""Transfer image bits from Glance or a known source location to the
supplied destination filepath.
:param context: The `cyborg.context.RequestContext` object for the
request
:param id_or_uri: A UUID identifier or an image URI to look up image
information for.
:param data: A file object to use in downloading image data.
:param dest_path: Filepath to transfer image bits to.
Note that because of the poor design of the
`glance.ImageService.download` method, the function returns different
things depending on what arguments are passed to it. If a data argument
is supplied but no dest_path is specified (only done in the XenAPI virt
driver's image.utils module) then None is returned from the method. If
the data argument is not specified but a destination path *is*
specified, then a writeable file handle to the destination path is
constructed in the method and the image bits written to that file, and
again, None is returned from the method. If no data argument is
supplied and no dest_path argument is supplied (VMWare and XenAPI virt
drivers), then the method returns an iterator to the image bits that
the caller uses to write to wherever location it wants. Finally, if the
allow_direct_url_schemes CONF option is set to something, then the
cyborg.image.download modules are used to attempt to do an SCP copy of
the image bits from a file location to the dest_path and None is
returned after retrying one or more download locations.
I think the above points to just how hacky/wacky all of this code is,
and the reason it needs to be cleaned up and standardized across the
virt driver callers.
"""
session, image_id = self._get_session_and_image_id(context, id_or_uri)
return session.download(context, image_id, data=data,
dst_path=dest_path)

View File

@ -0,0 +1,54 @@
# Copyright 2013 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
import stevedore.driver
import stevedore.extension
LOG = logging.getLogger(__name__)
def load_transfer_modules():
module_dictionary = {}
ex = stevedore.extension.ExtensionManager('cyborg.image.download.modules')
for module_name in ex.names():
mgr = stevedore.driver.DriverManager(
namespace='cyborg.image.download.modules',
name=module_name,
invoke_on_load=False)
schemes_list = mgr.driver.get_schemes()
for scheme in schemes_list:
if scheme in module_dictionary:
LOG.error('%(scheme)s is registered as a module twice. '
'%(module_name)s is not being used.',
{'scheme': scheme,
'module_name': module_name})
else:
module_dictionary[scheme] = mgr.driver
if module_dictionary:
LOG.warning('The cyborg.image.download.modules extension point is '
'deprecated for removal starting in the 17.0.0 Queens '
'release and may be removed as early as the 18.0.0 Rocky '
'release. It is not maintained and there is no indication '
'of its use in production clouds. If you are using this '
'extension point, please make the cyborg development team '
'aware by contacting us in the #openstack-cyborg freenode '
'IRC channel or on the openstack-dev mailing list.')
return module_dictionary

707
cyborg/image/glance.py Normal file
View File

@ -0,0 +1,707 @@
# Copyright 2010 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Implementation of an image service that uses Glance as the backend."""
from __future__ import absolute_import
import copy
import inspect
import itertools
import os
import random
import re
import stat
import sys
import time
import cryptography
from cursive import exception as cursive_exception
from cursive import signature_utils
import glanceclient
import glanceclient.exc
from glanceclient.v2 import schemas
from keystoneauth1 import loading as ks_loading
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import excutils
from oslo_utils import timeutils
import six
from six.moves import range
import six.moves.urllib.parse as urlparse
import cyborg.conf
from cyborg.common import exception
import cyborg.image.download as image_xfers
from cyborg import objects
from cyborg.objects import fields
from cyborg import service_auth
from cyborg.common import utils
LOG = logging.getLogger(__name__)
CONF = cyborg.conf.CONF
_SESSION = None
def _session_and_auth(context):
# Session is cached, but auth needs to be pulled from context each time.
global _SESSION
if not _SESSION:
_SESSION = ks_loading.load_session_from_conf_options(
CONF, cyborg.conf.glance.glance_group.name)
auth = service_auth.get_auth_plugin(context)
return _SESSION, auth
def _glanceclient_from_endpoint(context, endpoint, version):
sess, auth = _session_and_auth(context)
return glanceclient.Client(version, session=sess, auth=auth,
endpoint_override=endpoint,
global_request_id=context.global_id)
def generate_glance_url(context):
"""Return a random glance url from the api servers we know about."""
return next(get_api_servers(context))
def _endpoint_from_image_ref(image_href):
"""Return the image_ref and guessed endpoint from an image url.
:param image_href: href of an image
:returns: a tuple of the form (image_id, endpoint_url)
"""
parts = image_href.split('/')
image_id = parts[-1]
# the endpoint is everything in the url except the last 3 bits
# which are version, 'images', and image_id
endpoint = '/'.join(parts[:-3])
return (image_id, endpoint)
def generate_identity_headers(context, status='Confirmed'):
return {
'X-Auth-Token': getattr(context, 'auth_token', None),
'X-User-Id': getattr(context, 'user_id', None),
'X-Tenant-Id': getattr(context, 'project_id', None),
'X-Roles': ','.join(getattr(context, 'roles', [])),
'X-Identity-Status': status,
}
def get_api_servers(context):
"""Shuffle a list of service endpoints and return an iterator that will
cycle through the list, looping around to the beginning if necessary.
"""
# NOTE(efried): utils.get_ksa_adapter().get_endpoint() is the preferred
# mechanism for endpoint discovery. Only use `api_servers` if you really
# need to shuffle multiple endpoints.
if CONF.glance.api_servers:
api_servers = CONF.glance.api_servers
random.shuffle(api_servers)
else:
sess, auth = _session_and_auth(context)
ksa_adap = utils.get_ksa_adapter(
cyborg.conf.glance.DEFAULT_SERVICE_TYPE,
ksa_auth=auth, ksa_session=sess,
min_version='2.0', max_version='2.latest')
endpoint = utils.get_endpoint(ksa_adap)
if endpoint:
# NOTE(mriedem): Due to python-glanceclient bug 1707995 we have
# to massage the endpoint URL otherwise it won't work properly.
# We can't use glanceclient.common.utils.strip_version because
# of bug 1748009.
endpoint = re.sub(r'/v\d+(\.\d+)?/?$', '/', endpoint)
api_servers = [endpoint]
return itertools.cycle(api_servers)
class GlanceClientWrapper(object):
"""Glance client wrapper class that implements retries."""
def __init__(self, context=None, endpoint=None):
version = 2
if endpoint is not None:
self.client = self._create_static_client(context,
endpoint,
version)
else:
self.client = None
self.api_servers = None
def _create_static_client(self, context, endpoint, version):
"""Create a client that we'll use for every call."""
self.api_server = str(endpoint)
return _glanceclient_from_endpoint(context, endpoint, version)
def _create_onetime_client(self, context, version):
"""Create a client that will be used for one call."""
if self.api_servers is None:
self.api_servers = get_api_servers(context)
self.api_server = next(self.api_servers)
return _glanceclient_from_endpoint(context, self.api_server, version)
def call(self, context, version, method, *args, **kwargs):
"""Call a glance client method. If we get a connection error,
retry the request according to CONF.glance.num_retries.
"""
retry_excs = (glanceclient.exc.ServiceUnavailable,
glanceclient.exc.InvalidEndpoint,
glanceclient.exc.CommunicationError)
num_attempts = 1 + CONF.glance.num_retries
for attempt in range(1, num_attempts + 1):
client = self.client or self._create_onetime_client(context,
version)
try:
controller = getattr(client,
kwargs.pop('controller', 'images'))
result = getattr(controller, method)(*args, **kwargs)
if inspect.isgenerator(result):
# Convert generator results to a list, so that we can
# catch any potential exceptions now and retry the call.
return list(result)
return result
except retry_excs as e:
if attempt < num_attempts:
extra = "retrying"
else:
extra = 'done trying'
LOG.exception("Error contacting glance server "
"'%(server)s' for '%(method)s', "
"%(extra)s.",
{'server': self.api_server,
'method': method, 'extra': extra})
if attempt == num_attempts:
raise exception.GlanceConnectionFailed(
server=str(self.api_server), reason=six.text_type(e))
time.sleep(1)
class GlanceImageServiceV2(object):
"""Provides storage and retrieval of disk image objects within Glance."""
def __init__(self, client=None):
self._client = client or GlanceClientWrapper()
# NOTE(jbresnah) build the table of download handlers at the beginning
# so that operators can catch errors at load time rather than whenever
# a user attempts to use a module. Note this cannot be done in glance
# space when this python module is loaded because the download module
# may require configuration options to be parsed.
self._download_handlers = {}
download_modules = image_xfers.load_transfer_modules()
for scheme, mod in download_modules.items():
if scheme not in CONF.glance.allowed_direct_url_schemes:
continue
try:
self._download_handlers[scheme] = mod.get_download_handler()
except Exception as ex:
LOG.error('When loading the module %(module_str)s the '
'following error occurred: %(ex)s',
{'module_str': str(mod), 'ex': ex})
@staticmethod
def _safe_fsync(fh):
"""Performs os.fsync on a filehandle only if it is supported.
fsync on a pipe, FIFO, or socket raises OSError with EINVAL. This
method discovers whether the target filehandle is one of these types
and only performs fsync if it isn't.
:param fh: Open filehandle (not a path or fileno) to maybe fsync.
"""
fileno = fh.fileno()
mode = os.fstat(fileno).st_mode
# A pipe answers True to S_ISFIFO
if not any(check(mode) for check in (stat.S_ISFIFO, stat.S_ISSOCK)):
os.fsync(fileno)
def download(self, context, image_id, data=None, dst_path=None):
"""Calls out to Glance for data and writes data."""
if CONF.glance.allowed_direct_url_schemes and dst_path is not None:
image = self.show(context, image_id, include_locations=True)
for entry in image.get('locations', []):
loc_url = entry['url']
loc_meta = entry['metadata']
o = urlparse.urlparse(loc_url)
xfer_mod = self._get_transfer_module(o.scheme)
if xfer_mod:
try:
xfer_mod.download(context, o, dst_path, loc_meta)
LOG.info("Successfully transferred using %s", o.scheme)
return
except Exception:
LOG.exception("Download image error")
try:
image_chunks = self._client.call(context, 2, 'data', image_id)
except Exception:
_reraise_translated_image_exception(image_id)
if image_chunks.wrapped is None:
# None is a valid return value, but there's nothing we can do with
# a image with no associated data
raise exception.ImageUnacceptable(image_id=image_id,
reason='Image has no \
associated data')
# Retrieve properties for verification of Glance image signature
verifier = None
if CONF.glance.verify_glance_signatures:
image_meta_dict = self.show(context, image_id,
include_locations=False)
image_meta = objects.ImageMeta.from_dict(image_meta_dict)
img_signature = image_meta.properties.get('img_signature')
img_sig_hash_method = image_meta.properties.get(
'img_signature_hash_method'
)
img_sig_cert_uuid = image_meta.properties.get(
'img_signature_certificate_uuid'
)
img_sig_key_type = image_meta.properties.get(
'img_signature_key_type'
)
try:
verifier = signature_utils.get_verifier(
context=context,
img_signature_certificate_uuid=img_sig_cert_uuid,
img_signature_hash_method=img_sig_hash_method,
img_signature=img_signature,
img_signature_key_type=img_sig_key_type,
)
except cursive_exception.SignatureVerificationError:
with excutils.save_and_reraise_exception():
LOG.error('Image signature verification failed '
'for image: %s', image_id)
close_file = False
if data is None and dst_path:
data = open(dst_path, 'wb')
close_file = True
if data is None:
# Perform image signature verification
if verifier:
try:
for chunk in image_chunks:
verifier.update(chunk)
verifier.verify()
LOG.info('Image signature verification succeeded '
'for image: %s', image_id)
except cryptography.exceptions.InvalidSignature:
with excutils.save_and_reraise_exception():
LOG.error('Image signature verification failed '
'for image: %s', image_id)
return image_chunks
else:
try:
for chunk in image_chunks:
if verifier:
verifier.update(chunk)
data.write(chunk)
if verifier:
verifier.verify()
LOG.info('Image signature verification succeeded '
'for image %s', image_id)
except cryptography.exceptions.InvalidSignature:
data.truncate(0)
with excutils.save_and_reraise_exception():
LOG.error('Image signature verification failed '
'for image: %s', image_id)
except Exception as ex:
with excutils.save_and_reraise_exception():
LOG.error("Error writing to %(path)s: %(exception)s",
{'path': dst_path, 'exception': ex})
finally:
if close_file:
# Ensure that the data is pushed all the way down to
# persistent storage. This ensures that in the event of a
# subsequent host crash we don't have running instances
# using a corrupt backing file.
data.flush()
self._safe_fsync(data)
data.close()
def _extract_query_params(params):
_params = {}
accepted_params = ('filters', 'marker', 'limit',
'page_size', 'sort_key', 'sort_dir')
for param in accepted_params:
if params.get(param):
_params[param] = params.get(param)
# ensure filters is a dict
_params.setdefault('filters', {})
# NOTE(vish): don't filter out private images
_params['filters'].setdefault('is_public', 'none')
return _params
def _extract_query_params_v2(params):
_params = {}
accepted_params = ('filters', 'marker', 'limit',
'page_size', 'sort_key', 'sort_dir')
for param in accepted_params:
if params.get(param):
_params[param] = params.get(param)
# ensure filters is a dict
_params.setdefault('filters', {})
# NOTE(vish): don't filter out private images
_params['filters'].setdefault('is_public', 'none')
# adopt filters to be accepted by glance v2 api
filters = _params['filters']
new_filters = {}
for filter_ in filters:
# remove 'property-' prefix from filters by custom properties
if filter_.startswith('property-'):
new_filters[filter_.lstrip('property-')] = filters[filter_]
elif filter_ == 'changes-since':
# convert old 'changes-since' into new 'updated_at' filter
updated_at = 'gte:' + filters['changes-since']
new_filters['updated_at'] = updated_at
elif filter_ == 'is_public':
# convert old 'is_public' flag into 'visibility' filter
# omit the filter if is_public is None
is_public = filters['is_public']
if is_public.lower() in ('true', '1'):
new_filters['visibility'] = 'public'
elif is_public.lower() in ('false', '0'):
new_filters['visibility'] = 'private'
else:
new_filters[filter_] = filters[filter_]
_params['filters'] = new_filters
return _params
def _is_image_available(context, image):
"""Check image availability.
This check is needed in case cyborg and Glance are deployed
without authentication turned on.
"""
# The presence of an auth token implies this is an authenticated
# request and we need not handle the noauth use-case.
if hasattr(context, 'auth_token') and context.auth_token:
return True
def _is_image_public(image):
# NOTE(jaypipes) V2 Glance API replaced the is_public attribute
# with a visibility attribute. We do this here to prevent the
# glanceclient for a V2 image model from throwing an
# exception from warlock when trying to access an is_public
# attribute.
if hasattr(image, 'visibility'):
return str(image.visibility).lower() == 'public'
else:
return image.is_public
if context.is_admin or _is_image_public(image):
return True
properties = image.properties
if context.project_id and ('owner_id' in properties):
return str(properties['owner_id']) == str(context.project_id)
if context.project_id and ('project_id' in properties):
return str(properties['project_id']) == str(context.project_id)
try:
user_id = properties['user_id']
except KeyError:
return False
return str(user_id) == str(context.user_id)
def _translate_to_glance(image_meta):
image_meta = _convert_to_string(image_meta)
image_meta = _remove_read_only(image_meta)
image_meta = _convert_to_v2(image_meta)
return image_meta
def _convert_to_v2(image_meta):
output = {}
for name, value in image_meta.items():
if name == 'properties':
for prop_name, prop_value in value.items():
# if allow_additional_image_properties is disabled we can't
# define kernel_id and ramdisk_id as None, so we have to omit
# these properties if they are not set.
if prop_name in ('kernel_id', 'ramdisk_id') and \
prop_value is not None and \
prop_value.strip().lower() in ('none', ''):
continue
# in glance only string and None property values are allowed,
# v1 client accepts any values and converts them to string,
# v2 doesn't - so we have to take care of it.
elif prop_value is None or isinstance(
prop_value, six.string_types):
output[prop_name] = prop_value
else:
output[prop_name] = str(prop_value)
elif name in ('min_ram', 'min_disk'):
output[name] = int(value)
elif name == 'is_public':
output['visibility'] = 'public' if value else 'private'
elif name in ('size', 'deleted'):
continue
else:
output[name] = value
return output
def _translate_from_glance(image, include_locations=False):
image_meta = _extract_attributes_v2(
image, include_locations=include_locations)
image_meta = _convert_timestamps_to_datetimes(image_meta)
image_meta = _convert_from_string(image_meta)
return image_meta
def _convert_timestamps_to_datetimes(image_meta):
"""Returns image with timestamp fields converted to datetime objects."""
for attr in ['created_at', 'updated_at', 'deleted_at']:
if image_meta.get(attr):
image_meta[attr] = timeutils.parse_isotime(image_meta[attr])
return image_meta
# NOTE(bcwaldon): used to store non-string data in glance metadata
def _json_loads(properties, attr):
prop = properties[attr]
if isinstance(prop, six.string_types):
properties[attr] = jsonutils.loads(prop)
def _json_dumps(properties, attr):
prop = properties[attr]
if not isinstance(prop, six.string_types):
properties[attr] = jsonutils.dumps(prop)
_CONVERT_PROPS = ('block_device_mapping', 'mappings')
def _convert(method, metadata):
metadata = copy.deepcopy(metadata)
properties = metadata.get('properties')
if properties:
for attr in _CONVERT_PROPS:
if attr in properties:
method(properties, attr)
return metadata
def _convert_from_string(metadata):
return _convert(_json_loads, metadata)
def _convert_to_string(metadata):
return _convert(_json_dumps, metadata)
def _extract_attributes(image, include_locations=False):
# TODO(mfedosin): Remove this function once we move to glance V2
# completely.
# NOTE(hdd): If a key is not found, base.Resource.__getattr__() may perform
# a get(), resulting in a useless request back to glance. This list is
# therefore sorted, with dependent attributes as the end
# 'deleted_at' depends on 'deleted'
# 'checksum' depends on 'status' == 'active'
IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner',
'container_format', 'status', 'id',
'name', 'created_at', 'updated_at',
'deleted', 'deleted_at', 'checksum',
'min_disk', 'min_ram', 'is_public',
'direct_url', 'locations']
queued = getattr(image, 'status') == 'queued'
queued_exclude_attrs = ['disk_format', 'container_format']
include_locations_attrs = ['direct_url', 'locations']
output = {}
for attr in IMAGE_ATTRIBUTES:
if attr == 'deleted_at' and not output['deleted']:
output[attr] = None
elif attr == 'checksum' and output['status'] != 'active':
output[attr] = None
# image may not have 'name' attr
elif attr == 'name':
output[attr] = getattr(image, attr, None)
# NOTE(liusheng): queued image may not have these attributes and 'name'
elif queued and attr in queued_exclude_attrs:
output[attr] = getattr(image, attr, None)
# NOTE(mriedem): Only get location attrs if including locations.
elif attr in include_locations_attrs:
if include_locations:
output[attr] = getattr(image, attr, None)
# NOTE(mdorman): 'size' attribute must not be 'None', so use 0 instead
elif attr == 'size':
# NOTE(mriedem): A snapshot image may not have the size attribute
# set so default to 0.
output[attr] = getattr(image, attr, 0) or 0
else:
# NOTE(xarses): Anything that is caught with the default value
# will result in an additional lookup to glance for said attr.
# Notable attributes that could have this issue:
# disk_format, container_format, name, deleted, checksum
output[attr] = getattr(image, attr, None)
output['properties'] = getattr(image, 'properties', {})
return output
def _extract_attributes_v2(image, include_locations=False):
include_locations_attrs = ['direct_url', 'locations']
omit_attrs = ['self', 'schema', 'protected', 'virtual_size', 'file',
'tags']
raw_schema = image.schema
schema = schemas.Schema(raw_schema)
output = {'properties': {}, 'deleted': False, 'deleted_at': None,
'disk_format': None, 'container_format': None, 'name': None,
'checksum': None}
for name, value in image.items():
if (name in omit_attrs
or name in include_locations_attrs and not include_locations):
continue
elif name == 'visibility':
output['is_public'] = value == 'public'
elif name == 'size' and value is None:
output['size'] = 0
elif schema.is_base_property(name):
output[name] = value
else:
output['properties'][name] = value
return output
def _remove_read_only(image_meta):
IMAGE_ATTRIBUTES = ['status', 'updated_at', 'created_at', 'deleted_at']
output = copy.deepcopy(image_meta)
for attr in IMAGE_ATTRIBUTES:
if attr in output:
del output[attr]
return output
def _reraise_translated_image_exception(image_id):
"""Transform the exception for the image but keep its traceback intact."""
exc_type, exc_value, exc_trace = sys.exc_info()
new_exc = _translate_image_exception(image_id, exc_value)
six.reraise(type(new_exc), new_exc, exc_trace)
def _reraise_translated_exception():
"""Transform the exception but keep its traceback intact."""
exc_type, exc_value, exc_trace = sys.exc_info()
new_exc = _translate_plain_exception(exc_value)
six.reraise(type(new_exc), new_exc, exc_trace)
def _translate_image_exception(image_id, exc_value):
if isinstance(exc_value, (glanceclient.exc.Forbidden,
glanceclient.exc.Unauthorized)):
return exception.ImageNotAuthorized(image_id=image_id)
if isinstance(exc_value, glanceclient.exc.NotFound):
return exception.ImageNotFound(image_id=image_id)
if isinstance(exc_value, glanceclient.exc.BadRequest):
return exception.ImageBadRequest(image_id=image_id,
response=six.text_type(exc_value))
return exc_value
def _translate_plain_exception(exc_value):
if isinstance(exc_value, (glanceclient.exc.Forbidden,
glanceclient.exc.Unauthorized)):
return exception.Forbidden(six.text_type(exc_value))
if isinstance(exc_value, glanceclient.exc.NotFound):
return exception.NotFound(six.text_type(exc_value))
if isinstance(exc_value, glanceclient.exc.BadRequest):
return exception.Invalid(six.text_type(exc_value))
return exc_value
def get_remote_image_service(context, image_href):
"""Create an image_service and parse the id from the given image_href.
The image_href param can be an href of the form