Added Glance Client for Image downloading
According to sean mooney's comments in https://review.openstack.org/#/c/596507/ I am splitting into 2 patches For now, please add following session in cyborg.conf [glance] project_domain_name = Default project_name = service user_domain_name = Default password = {your_password} username = placement auth_url = http://{host_ip}/identity auth_type = password auth_section = keystone_authtoken api_servers = http://{host_ip}/image Change-Id: I3a0249c7ac8ba5fd3c20ae6402dcc2f9e9bcfe95
This commit is contained in:
Li Liu
parent
715f03953a
commit
156c9f18e5
@ -305,3 +305,25 @@ class QuotaResourceUnknown(QuotaNotFound):
|
||||
|
||||
class InvalidReservationExpiration(Invalid):
|
||||
message = _("Invalid reservation expiration %(expire)s.")
|
||||
|
||||
|
||||
class GlanceConnectionFailed(CyborgException):
|
||||
msg_fmt = _("Connection to glance host %(server)s failed: "
|
||||
"%(reason)s")
|
||||
|
||||
|
||||
class ImageUnacceptable(Invalid):
|
||||
msg_fmt = _("Image %(image_id)s is unacceptable: %(reason)s")
|
||||
|
||||
|
||||
class ImageNotAuthorized(CyborgException):
|
||||
msg_fmt = _("Not authorized for image %(image_id)s.")
|
||||
|
||||
|
||||
class ImageNotFound(NotFound):
|
||||
msg_fmt = _("Image %(image_id)s could not be found.")
|
||||
|
||||
|
||||
class ImageBadRequest(Invalid):
|
||||
msg_fmt = _("Request of image %(image_id)s got BadRequest response: "
|
||||
"%(response)s")
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
# under the License.
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_context import context as cyborg_context
|
||||
from cyborg import context as cyborg_context
|
||||
import oslo_messaging as messaging
|
||||
from oslo_messaging.rpc import dispatcher
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
# under the License.
|
||||
|
||||
from oslo_concurrency import processutils
|
||||
from oslo_context import context
|
||||
from cyborg import context
|
||||
from oslo_log import log
|
||||
import oslo_messaging as messaging
|
||||
from oslo_service import service
|
||||
|
||||
@ -108,3 +108,57 @@ def get_ksa_adapter(service_type, ksa_auth=None, ksa_session=None,
|
||||
return ks_loading.load_adapter_from_conf_options(
|
||||
CONF, confgrp, session=ksa_session, auth=ksa_auth,
|
||||
min_version=min_version, max_version=max_version)
|
||||
|
||||
|
||||
def get_endpoint(ksa_adapter):
|
||||
"""Get the endpoint URL represented by a keystoneauth1 Adapter.
|
||||
|
||||
This method is equivalent to what
|
||||
|
||||
ksa_adapter.get_endpoint()
|
||||
|
||||
should do, if it weren't for a panoply of bugs.
|
||||
|
||||
:param ksa_adapter: keystoneauth1.adapter.Adapter, appropriately set up
|
||||
with an endpoint_override; or service_type, interface
|
||||
(list) and auth/service_catalog.
|
||||
:return: String endpoint URL.
|
||||
:raise EndpointNotFound: If endpoint discovery fails.
|
||||
"""
|
||||
# TODO(efried): This will be unnecessary once bug #1707993 is fixed.
|
||||
# (At least for the non-image case, until 1707995 is fixed.)
|
||||
if ksa_adapter.endpoint_override:
|
||||
return ksa_adapter.endpoint_override
|
||||
# TODO(efried): Remove this once bug #1707995 is fixed.
|
||||
if ksa_adapter.service_type == 'image':
|
||||
try:
|
||||
# LOG.warning(ksa_adapter.__dict__)
|
||||
return ksa_adapter.get_endpoint_data().catalog_url
|
||||
except AttributeError:
|
||||
# ksa_adapter.auth is a _ContextAuthPlugin, which doesn't have
|
||||
# get_endpoint_data. Fall through to using get_endpoint().
|
||||
pass
|
||||
# TODO(efried): The remainder of this method reduces to
|
||||
# TODO(efried): return ksa_adapter.get_endpoint()
|
||||
# TODO(efried): once bug #1709118 is fixed.
|
||||
# NOTE(efried): Id9bd19cca68206fc64d23b0eaa95aa3e5b01b676 may also do the
|
||||
# trick, once it's in a ksa release.
|
||||
# The EndpointNotFound exception happens when _ContextAuthPlugin is in play
|
||||
# because its get_endpoint() method isn't yet set up to handle interface as
|
||||
# a list. (It could also happen with a real auth if the endpoint isn't
|
||||
# there; but that's covered below.)
|
||||
try:
|
||||
return ksa_adapter.get_endpoint()
|
||||
except ks_exc.EndpointNotFound:
|
||||
pass
|
||||
|
||||
interfaces = list(ksa_adapter.interface)
|
||||
for interface in interfaces:
|
||||
ksa_adapter.interface = interface
|
||||
try:
|
||||
return ksa_adapter.get_endpoint()
|
||||
except ks_exc.EndpointNotFound:
|
||||
pass
|
||||
raise ks_exc.EndpointNotFound(
|
||||
"Could not find requested endpoint for any of the following "
|
||||
"interfaces: %s" % interfaces)
|
||||
|
||||
@ -18,7 +18,9 @@ from oslo_config import cfg
|
||||
from cyborg.conf import api
|
||||
from cyborg.conf import database
|
||||
from cyborg.conf import default
|
||||
|
||||
from cyborg.conf import service_token
|
||||
from cyborg.conf import glance
|
||||
from cyborg.conf import keystone
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
@ -26,3 +28,6 @@ api.register_opts(CONF)
|
||||
database.register_opts(CONF)
|
||||
default.register_opts(CONF)
|
||||
default.register_placement_opts(CONF)
|
||||
service_token.register_opts(CONF)
|
||||
glance.register_opts(CONF)
|
||||
keystone.register_opts(CONF)
|
||||
|
||||
171
cyborg/conf/glance.py
Normal file
171
cyborg/conf/glance.py
Normal file
@ -0,0 +1,171 @@
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from keystoneauth1 import loading as ks_loading
|
||||
from oslo_config import cfg
|
||||
|
||||
from cyborg.conf import utils as confutils
|
||||
|
||||
|
||||
DEFAULT_SERVICE_TYPE = 'image'
|
||||
|
||||
glance_group = cfg.OptGroup(
|
||||
'glance',
|
||||
title='Glance Options',
|
||||
help='Configuration options for the Image service')
|
||||
|
||||
glance_opts = [
|
||||
# NOTE(sdague/efried): there is intentionally no default here. This
|
||||
# requires configuration if ksa adapter config is not used.
|
||||
cfg.ListOpt('api_servers',
|
||||
help="""
|
||||
List of glance api servers endpoints available to cyborg.
|
||||
|
||||
https is used for ssl-based glance api servers.
|
||||
|
||||
NOTE: The preferred mechanism for endpoint discovery is via keystoneauth1
|
||||
loading options. Only use api_servers if you need multiple endpoints and are
|
||||
unable to use a load balancer for some reason.
|
||||
|
||||
Possible values:
|
||||
|
||||
* A list of any fully qualified url of the form "scheme://hostname:port[/path]"
|
||||
(i.e. "http://10.0.1.0:9292" or "https://my.glance.server/image").
|
||||
"""),
|
||||
cfg.IntOpt('num_retries',
|
||||
default=0,
|
||||
min=0,
|
||||
help="""
|
||||
Enable glance operation retries.
|
||||
|
||||
Specifies the number of retries when uploading / downloading
|
||||
an image to / from glance. 0 means no retries.
|
||||
"""),
|
||||
cfg.ListOpt('allowed_direct_url_schemes',
|
||||
default=[],
|
||||
deprecated_for_removal=True,
|
||||
deprecated_since='17.0.0',
|
||||
deprecated_reason="""
|
||||
This was originally added for the 'cyborg.image.download.file' FileTransfer
|
||||
extension which was removed in the 16.0.0 Pike release. The
|
||||
'cyborg.image.download.modules' extension point is not maintained
|
||||
and there is no indication of its use in production clouds.
|
||||
""",
|
||||
help="""
|
||||
List of url schemes that can be directly accessed.
|
||||
|
||||
This option specifies a list of url schemes that can be downloaded
|
||||
directly via the direct_url. This direct_URL can be fetched from
|
||||
Image metadata which can be used by cyborg to get the
|
||||
image more efficiently. cyborg-compute could benefit from this by
|
||||
invoking a copy when it has access to the same file system as glance.
|
||||
|
||||
Possible values:
|
||||
|
||||
* [file], Empty list (default)
|
||||
"""),
|
||||
cfg.BoolOpt('verify_glance_signatures',
|
||||
default=False,
|
||||
help="""
|
||||
Enable image signature verification.
|
||||
|
||||
cyborg uses the image signature metadata from glance and verifies the signature
|
||||
of a signed image while downloading that image. If the image signature cannot
|
||||
be verified or if the image signature metadata is either incomplete or
|
||||
unavailable, then cyborg will not boot the image and instead will place the
|
||||
instance into an error state. This provides end users with stronger assurances
|
||||
of the integrity of the image data they are using to create servers.
|
||||
|
||||
Related options:
|
||||
|
||||
* The options in the `key_manager` group, as the key_manager is used
|
||||
for the signature validation.
|
||||
* Both enable_certificate_validation and default_trusted_certificate_ids
|
||||
below depend on this option being enabled.
|
||||
"""),
|
||||
cfg.BoolOpt('enable_certificate_validation',
|
||||
default=False,
|
||||
deprecated_for_removal=True,
|
||||
deprecated_since='16.0.0',
|
||||
deprecated_reason="""
|
||||
This option is intended to ease the transition for deployments leveraging
|
||||
image signature verification. The intended state long-term is for signature
|
||||
verification and certificate validation to always happen together.
|
||||
""",
|
||||
help="""
|
||||
Enable certificate validation for image signature verification.
|
||||
|
||||
During image signature verification cyborg will first verify the validity of
|
||||
the image's signing certificate using the set of trusted certificates
|
||||
associated with the instance. If certificate validation fails, signature
|
||||
verification will not be performed and the instance will be placed into an
|
||||
error state. This provides end users with stronger assurances that the image
|
||||
data is unmodified and trustworthy. If left disabled, image signature
|
||||
verification can still occur but the end user will not have any assurance that
|
||||
the signing certificate used to generate the image signature is still
|
||||
trustworthy.
|
||||
|
||||
Related options:
|
||||
|
||||
* This option only takes effect if verify_glance_signatures is enabled.
|
||||
* The value of default_trusted_certificate_ids may be used when this option
|
||||
is enabled.
|
||||
"""),
|
||||
cfg.ListOpt('default_trusted_certificate_ids',
|
||||
default=[],
|
||||
help="""
|
||||
List of certificate IDs for certificates that should be trusted.
|
||||
|
||||
May be used as a default list of trusted certificate IDs for certificate
|
||||
validation. The value of this option will be ignored if the user provides a
|
||||
list of trusted certificate IDs with an instance API request. The value of
|
||||
this option will be persisted with the instance data if signature verification
|
||||
and certificate validation are enabled and if the user did not provide an
|
||||
alternative list. If left empty when certificate validation is enabled the
|
||||
user must provide a list of trusted certificate IDs otherwise certificate
|
||||
validation will fail.
|
||||
|
||||
Related options:
|
||||
|
||||
* The value of this option may be used if both verify_glance_signatures and
|
||||
enable_certificate_validation are enabled.
|
||||
"""),
|
||||
cfg.BoolOpt('debug',
|
||||
default=False,
|
||||
help='Enable or disable debug logging with glanceclient.')
|
||||
]
|
||||
|
||||
deprecated_ksa_opts = {
|
||||
'insecure': [cfg.DeprecatedOpt('api_insecure', group=glance_group.name)],
|
||||
'cafile': [cfg.DeprecatedOpt('ca_file', group="ssl")],
|
||||
'certfile': [cfg.DeprecatedOpt('cert_file', group="ssl")],
|
||||
'keyfile': [cfg.DeprecatedOpt('key_file', group="ssl")],
|
||||
}
|
||||
|
||||
|
||||
def register_opts(conf):
|
||||
conf.register_group(glance_group)
|
||||
conf.register_opts(glance_opts, group=glance_group)
|
||||
|
||||
confutils.register_ksa_opts(
|
||||
conf, glance_group, DEFAULT_SERVICE_TYPE, include_auth=False,
|
||||
deprecated_opts=deprecated_ksa_opts)
|
||||
|
||||
|
||||
def list_opts():
|
||||
return {glance_group: (
|
||||
glance_opts +
|
||||
ks_loading.get_session_conf_options() +
|
||||
confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE,
|
||||
deprecated_opts=deprecated_ksa_opts))}
|
||||
40
cyborg/conf/keystone.py
Normal file
40
cyborg/conf/keystone.py
Normal file
@ -0,0 +1,40 @@
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from keystoneauth1 import loading as ks_loading
|
||||
from oslo_config import cfg
|
||||
|
||||
from cyborg.conf import utils as confutils
|
||||
|
||||
|
||||
DEFAULT_SERVICE_TYPE = 'identity'
|
||||
|
||||
keystone_group = cfg.OptGroup(
|
||||
'keystone',
|
||||
title='Keystone Options',
|
||||
help='Configuration options for the identity service')
|
||||
|
||||
|
||||
def register_opts(conf):
|
||||
conf.register_group(keystone_group)
|
||||
confutils.register_ksa_opts(conf, keystone_group.name,
|
||||
DEFAULT_SERVICE_TYPE, include_auth=False)
|
||||
|
||||
|
||||
def list_opts():
|
||||
return {
|
||||
keystone_group: (
|
||||
ks_loading.get_session_conf_options() +
|
||||
confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE)
|
||||
)
|
||||
}
|
||||
54
cyborg/conf/service_token.py
Normal file
54
cyborg/conf/service_token.py
Normal file
@ -0,0 +1,54 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from keystoneauth1 import loading as ks_loading
|
||||
from oslo_config import cfg
|
||||
|
||||
SERVICE_USER_GROUP = 'service_user'
|
||||
|
||||
service_user = cfg.OptGroup(
|
||||
SERVICE_USER_GROUP,
|
||||
title='Service token authentication type options',
|
||||
help="""
|
||||
Configuration options for service to service authentication using a service
|
||||
token. These options allow sending a service token along with the user's token
|
||||
when contacting external REST APIs.
|
||||
"""
|
||||
)
|
||||
|
||||
service_user_opts = [
|
||||
cfg.BoolOpt('send_service_user_token',
|
||||
default=False,
|
||||
help="""
|
||||
When True, if sending a user token to a REST API, also send a service token.
|
||||
"""),
|
||||
]
|
||||
|
||||
|
||||
def register_opts(conf):
|
||||
conf.register_group(service_user)
|
||||
conf.register_opts(service_user_opts, group=service_user)
|
||||
|
||||
ks_loading.register_session_conf_options(conf, SERVICE_USER_GROUP)
|
||||
ks_loading.register_auth_conf_options(conf, SERVICE_USER_GROUP)
|
||||
|
||||
|
||||
def list_opts():
|
||||
return {
|
||||
service_user: (
|
||||
service_user_opts +
|
||||
ks_loading.get_session_conf_options() +
|
||||
ks_loading.get_auth_common_conf_options() +
|
||||
ks_loading.get_auth_plugin_conf_options('password') +
|
||||
ks_loading.get_auth_plugin_conf_options('v2password') +
|
||||
ks_loading.get_auth_plugin_conf_options('v3password'))
|
||||
}
|
||||
171
cyborg/context.py
Normal file
171
cyborg/context.py
Normal file
@ -0,0 +1,171 @@
|
||||
# Copyright 2011 OpenStack Foundation
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
from keystoneauth1.access import service_catalog as ksa_service_catalog
|
||||
from keystoneauth1 import plugin
|
||||
from oslo_context import context
|
||||
from oslo_db.sqlalchemy import enginefacade
|
||||
from oslo_utils import timeutils
|
||||
import six
|
||||
|
||||
|
||||
class _ContextAuthPlugin(plugin.BaseAuthPlugin):
|
||||
"""A keystoneauth auth plugin that uses the values from the Context.
|
||||
|
||||
Ideally we would use the plugin provided by auth_token middleware however
|
||||
this plugin isn't serialized yet so we construct one from the serialized
|
||||
auth data.
|
||||
"""
|
||||
|
||||
def __init__(self, auth_token, sc):
|
||||
super(_ContextAuthPlugin, self).__init__()
|
||||
|
||||
self.auth_token = auth_token
|
||||
self.service_catalog = ksa_service_catalog.ServiceCatalogV2(sc)
|
||||
|
||||
def get_token(self, *args, **kwargs):
|
||||
return self.auth_token
|
||||
|
||||
def get_endpoint(self, session, service_type=None, interface=None,
|
||||
region_name=None, service_name=None, **kwargs):
|
||||
return self.service_catalog.url_for(service_type=service_type,
|
||||
service_name=service_name,
|
||||
interface=interface,
|
||||
region_name=region_name)
|
||||
|
||||
|
||||
@enginefacade.transaction_context_provider
|
||||
class RequestContext(context.RequestContext):
|
||||
"""Security context and request information.
|
||||
|
||||
Represents the user taking a given action within the system.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, user_id=None, project_id=None, is_admin=None,
|
||||
read_deleted="no", remote_address=None, timestamp=None,
|
||||
quota_class=None, service_catalog=None,
|
||||
user_auth_plugin=None, **kwargs):
|
||||
""":param read_deleted: 'no' indicates deleted records are hidden,
|
||||
'yes' indicates deleted records are visible,
|
||||
'only' indicates that *only* deleted records are visible.
|
||||
|
||||
:param overwrite: Set to False to ensure that the greenthread local
|
||||
copy of the index is not overwritten.
|
||||
|
||||
:param instance_lock_checked: This is not used and will be removed
|
||||
in a future release.
|
||||
|
||||
:param user_auth_plugin: The auth plugin for the current request's
|
||||
authentication data.
|
||||
"""
|
||||
if user_id:
|
||||
kwargs['user_id'] = user_id
|
||||
if project_id:
|
||||
kwargs['project_id'] = project_id
|
||||
|
||||
super(RequestContext, self).__init__(is_admin=is_admin, **kwargs)
|
||||
|
||||
self.read_deleted = read_deleted
|
||||
self.remote_address = remote_address
|
||||
if not timestamp:
|
||||
timestamp = timeutils.utcnow()
|
||||
if isinstance(timestamp, six.string_types):
|
||||
timestamp = timeutils.parse_strtime(timestamp)
|
||||
self.timestamp = timestamp
|
||||
|
||||
if service_catalog:
|
||||
# Only include required parts of service_catalog
|
||||
self.service_catalog = [s for s in service_catalog
|
||||
if s.get('type') in ('image')]
|
||||
else:
|
||||
# if list is empty or none
|
||||
self.service_catalog = []
|
||||
|
||||
self.user_auth_plugin = user_auth_plugin
|
||||
# if self.is_admin is None:
|
||||
# self.is_admin = policy.check_is_admin(self)
|
||||
|
||||
def get_auth_plugin(self):
|
||||
if self.user_auth_plugin:
|
||||
return self.user_auth_plugin
|
||||
else:
|
||||
return _ContextAuthPlugin(self.auth_token, self.service_catalog)
|
||||
|
||||
|
||||
def get_context():
|
||||
"""A helper method to get a blank context.
|
||||
|
||||
Note that overwrite is False here so this context will not update the
|
||||
greenthread-local stored context that is used when logging.
|
||||
"""
|
||||
return RequestContext(user_id=None,
|
||||
project_id=None,
|
||||
is_admin=False,
|
||||
overwrite=False)
|
||||
|
||||
|
||||
def get_admin_context(read_deleted="no"):
|
||||
# NOTE(alaski): This method should only be used when an admin context is
|
||||
# necessary for the entirety of the context lifetime. If that's not the
|
||||
# case please use get_context(), or create the RequestContext manually, and
|
||||
# use context.elevated() where necessary. Some periodic tasks may use
|
||||
# get_admin_context so that their database calls are not filtered on
|
||||
# project_id.
|
||||
return RequestContext(user_id=None,
|
||||
project_id=None,
|
||||
is_admin=True,
|
||||
read_deleted=read_deleted,
|
||||
overwrite=False)
|
||||
|
||||
|
||||
def is_user_context(context):
|
||||
"""Indicates if the request context is a normal user."""
|
||||
if not context:
|
||||
return False
|
||||
if context.is_admin:
|
||||
return False
|
||||
if not context.user_id or not context.project_id:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def require_context(ctxt):
|
||||
"""Raise exception.Forbidden() if context is not a user or an
|
||||
admin context.
|
||||
"""
|
||||
if not ctxt.is_admin and not is_user_context(ctxt):
|
||||
raise exception.Forbidden()
|
||||
|
||||
|
||||
def authorize_project_context(context, project_id):
|
||||
"""Ensures a request has permission to access the given project."""
|
||||
if is_user_context(context):
|
||||
if not context.project_id:
|
||||
raise exception.Forbidden()
|
||||
elif context.project_id != project_id:
|
||||
raise exception.Forbidden()
|
||||
|
||||
|
||||
def authorize_user_context(context, user_id):
|
||||
"""Ensures a request has permission to access the given user."""
|
||||
if is_user_context(context):
|
||||
if not context.user_id:
|
||||
raise exception.Forbidden()
|
||||
elif context.user_id != user_id:
|
||||
raise exception.Forbidden()
|
||||
0
cyborg/image/__init__.py
Normal file
0
cyborg/image/__init__.py
Normal file
163
cyborg/image/api.py
Normal file
163
cyborg/image/api.py
Normal file
@ -0,0 +1,163 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Main abstraction layer for retrieving and storing information about acclerator
|
||||
images used by the cyborg agent layer.
|
||||
"""
|
||||
|
||||
from cyborg.image import glance
|
||||
from oslo_log import log
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class API(object):
|
||||
|
||||
"""Responsible for exposing a relatively stable internal API for other
|
||||
modules in Cyborg to retrieve information about acclerator images.
|
||||
"""
|
||||
|
||||
def _get_session_and_image_id(self, context, id_or_uri):
|
||||
"""Returns a tuple of (session, image_id). If the supplied `id_or_uri`
|
||||
is an image ID, then the default client session will be returned
|
||||
for the context's user, along with the image ID. If the supplied
|
||||
`id_or_uri` parameter is a URI, then a client session connecting to
|
||||
the URI's image service endpoint will be returned along with a
|
||||
parsed image ID from that URI.
|
||||
|
||||
:param context: The `cyborg.context.Context` object for the request
|
||||
:param id_or_uri: A UUID identifier or an image URI to look up image
|
||||
information for.
|
||||
"""
|
||||
return glance.get_remote_image_service(context, id_or_uri)
|
||||
|
||||
def _get_session(self, _context):
|
||||
"""Returns a client session that can be used to query for image
|
||||
information.
|
||||
|
||||
:param _context: The `cyborg.context.Context` object for the request
|
||||
"""
|
||||
return glance.get_default_image_service()
|
||||
|
||||
@staticmethod
|
||||
def generate_image_url(image_ref, context):
|
||||
"""Generate an image URL from an image_ref.
|
||||
|
||||
:param image_ref: The image ref to generate URL
|
||||
:param context: The `cyborg.context.Context` object for the request
|
||||
"""
|
||||
return "%s/images/%s" % (next(glance.get_api_servers(context)),
|
||||
image_ref)
|
||||
|
||||
def get_all(self, context, **kwargs):
|
||||
"""Retrieves all information records about all acclerator images
|
||||
available to show to the requesting user. If the requesting user is an
|
||||
admin, all images in an ACTIVE status are returned. If the requesting
|
||||
user is not an admin, the all public images and all private images
|
||||
that are owned by the requesting user in the ACTIVE status are
|
||||
returned.
|
||||
|
||||
:param context: The `cyborg.context.Context` object for the request
|
||||
:param kwargs: A dictionary of filter and pagination values that
|
||||
may be passed to the underlying image info driver.
|
||||
"""
|
||||
session = self._get_session(context)
|
||||
return session.detail(context, **kwargs)
|
||||
|
||||
def get(self, context, id_or_uri):
|
||||
"""Retrieves the information record for a single acclerator image.
|
||||
If the supplied identifier parameter is a UUID, the default driver will
|
||||
be used to return information about the image. If the supplied
|
||||
identifier is a URI, then the driver that matches that URI endpoint
|
||||
will be used to query for image information.
|
||||
|
||||
:param context: The `cyborg.context.Context` object for the request
|
||||
:param id_or_uri: A UUID identifier or an image URI to look up image
|
||||
information for.
|
||||
"""
|
||||
session, image_id = self._get_session_and_image_id(context, id_or_uri)
|
||||
return session.show(context, image_id,
|
||||
include_locations=False,
|
||||
show_deleted=False)
|
||||
|
||||
def update(self, context, id_or_uri, image_info,
|
||||
data=None, purge_props=False):
|
||||
"""Update the information about an image, optionally along with a file
|
||||
handle or bytestream iterator for image bits. If the optional file
|
||||
handle for updated image bits is supplied, the image may not have
|
||||
already uploaded bits for the image.
|
||||
|
||||
:param context: The `cyborg.context.Context` object for the request
|
||||
:param id_or_uri: A UUID identifier or an image URI to look up image
|
||||
information for.
|
||||
:param image_info: A dict of information about the image that is
|
||||
passed to the image registry.
|
||||
:param data: Optional file handle or bytestream iterator that is
|
||||
passed to backend storage.
|
||||
:param purge_props: Optional, defaults to False. If set, the backend
|
||||
image registry will clear all image properties
|
||||
and replace them the image properties supplied
|
||||
in the image_info dictionary's 'properties'
|
||||
collection.
|
||||
"""
|
||||
session, image_id = self._get_session_and_image_id(context, id_or_uri)
|
||||
return session.update(context, image_id, image_info, data=data,
|
||||
purge_props=purge_props)
|
||||
|
||||
def delete(self, context, id_or_uri):
|
||||
"""Delete the information about an image and mark the image bits for
|
||||
deletion.
|
||||
|
||||
:param context: The `cyborg.context.Context` object for the request
|
||||
:param id_or_uri: A UUID identifier or an image URI to look up image
|
||||
information for.
|
||||
"""
|
||||
session, image_id = self._get_session_and_image_id(context, id_or_uri)
|
||||
return session.delete(context, image_id)
|
||||
|
||||
def download(self, context, id_or_uri, data=None, dest_path=None):
|
||||
"""Transfer image bits from Glance or a known source location to the
|
||||
supplied destination filepath.
|
||||
|
||||
:param context: The `cyborg.context.RequestContext` object for the
|
||||
request
|
||||
:param id_or_uri: A UUID identifier or an image URI to look up image
|
||||
information for.
|
||||
:param data: A file object to use in downloading image data.
|
||||
:param dest_path: Filepath to transfer image bits to.
|
||||
|
||||
Note that because of the poor design of the
|
||||
`glance.ImageService.download` method, the function returns different
|
||||
things depending on what arguments are passed to it. If a data argument
|
||||
is supplied but no dest_path is specified (only done in the XenAPI virt
|
||||
driver's image.utils module) then None is returned from the method. If
|
||||
the data argument is not specified but a destination path *is*
|
||||
specified, then a writeable file handle to the destination path is
|
||||
constructed in the method and the image bits written to that file, and
|
||||
again, None is returned from the method. If no data argument is
|
||||
supplied and no dest_path argument is supplied (VMWare and XenAPI virt
|
||||
drivers), then the method returns an iterator to the image bits that
|
||||
the caller uses to write to wherever location it wants. Finally, if the
|
||||
allow_direct_url_schemes CONF option is set to something, then the
|
||||
cyborg.image.download modules are used to attempt to do an SCP copy of
|
||||
the image bits from a file location to the dest_path and None is
|
||||
returned after retrying one or more download locations.
|
||||
|
||||
I think the above points to just how hacky/wacky all of this code is,
|
||||
and the reason it needs to be cleaned up and standardized across the
|
||||
virt driver callers.
|
||||
"""
|
||||
|
||||
session, image_id = self._get_session_and_image_id(context, id_or_uri)
|
||||
return session.download(context, image_id, data=data,
|
||||
dst_path=dest_path)
|
||||
54
cyborg/image/download/__init__.py
Normal file
54
cyborg/image/download/__init__.py
Normal file
@ -0,0 +1,54 @@
|
||||
# Copyright 2013 Red Hat, Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_log import log as logging
|
||||
import stevedore.driver
|
||||
import stevedore.extension
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_transfer_modules():
|
||||
|
||||
module_dictionary = {}
|
||||
|
||||
ex = stevedore.extension.ExtensionManager('cyborg.image.download.modules')
|
||||
for module_name in ex.names():
|
||||
mgr = stevedore.driver.DriverManager(
|
||||
namespace='cyborg.image.download.modules',
|
||||
name=module_name,
|
||||
invoke_on_load=False)
|
||||
|
||||
schemes_list = mgr.driver.get_schemes()
|
||||
for scheme in schemes_list:
|
||||
if scheme in module_dictionary:
|
||||
LOG.error('%(scheme)s is registered as a module twice. '
|
||||
'%(module_name)s is not being used.',
|
||||
{'scheme': scheme,
|
||||
'module_name': module_name})
|
||||
else:
|
||||
module_dictionary[scheme] = mgr.driver
|
||||
|
||||
if module_dictionary:
|
||||
LOG.warning('The cyborg.image.download.modules extension point is '
|
||||
'deprecated for removal starting in the 17.0.0 Queens '
|
||||
'release and may be removed as early as the 18.0.0 Rocky '
|
||||
'release. It is not maintained and there is no indication '
|
||||
'of its use in production clouds. If you are using this '
|
||||
'extension point, please make the cyborg development team '
|
||||
'aware by contacting us in the #openstack-cyborg freenode '
|
||||
'IRC channel or on the openstack-dev mailing list.')
|
||||
|
||||
return module_dictionary
|
||||
707
cyborg/image/glance.py
Normal file
707
cyborg/image/glance.py
Normal file
@ -0,0 +1,707 @@
|
||||
# Copyright 2010 OpenStack Foundation
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Implementation of an image service that uses Glance as the backend."""
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
import itertools
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
|
||||
import cryptography
|
||||
from cursive import exception as cursive_exception
|
||||
from cursive import signature_utils
|
||||
import glanceclient
|
||||
import glanceclient.exc
|
||||
from glanceclient.v2 import schemas
|
||||
from keystoneauth1 import loading as ks_loading
|
||||
from oslo_log import log as logging
|
||||
from oslo_serialization import jsonutils
|
||||
from oslo_utils import excutils
|
||||
from oslo_utils import timeutils
|
||||
import six
|
||||
from six.moves import range
|
||||
import six.moves.urllib.parse as urlparse
|
||||
|
||||
import cyborg.conf
|
||||
from cyborg.common import exception
|
||||
import cyborg.image.download as image_xfers
|
||||
from cyborg import objects
|
||||
from cyborg.objects import fields
|
||||
from cyborg import service_auth
|
||||
from cyborg.common import utils
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = cyborg.conf.CONF
|
||||
|
||||
_SESSION = None
|
||||
|
||||
|
||||
def _session_and_auth(context):
|
||||
# Session is cached, but auth needs to be pulled from context each time.
|
||||
global _SESSION
|
||||
|
||||
if not _SESSION:
|
||||
_SESSION = ks_loading.load_session_from_conf_options(
|
||||
CONF, cyborg.conf.glance.glance_group.name)
|
||||
|
||||
auth = service_auth.get_auth_plugin(context)
|
||||
|
||||
return _SESSION, auth
|
||||
|
||||
|
||||
def _glanceclient_from_endpoint(context, endpoint, version):
|
||||
sess, auth = _session_and_auth(context)
|
||||
|
||||
return glanceclient.Client(version, session=sess, auth=auth,
|
||||
endpoint_override=endpoint,
|
||||
global_request_id=context.global_id)
|
||||
|
||||
|
||||
def generate_glance_url(context):
|
||||
"""Return a random glance url from the api servers we know about."""
|
||||
return next(get_api_servers(context))
|
||||
|
||||
|
||||
def _endpoint_from_image_ref(image_href):
|
||||
"""Return the image_ref and guessed endpoint from an image url.
|
||||
|
||||
:param image_href: href of an image
|
||||
:returns: a tuple of the form (image_id, endpoint_url)
|
||||
"""
|
||||
parts = image_href.split('/')
|
||||
image_id = parts[-1]
|
||||
# the endpoint is everything in the url except the last 3 bits
|
||||
# which are version, 'images', and image_id
|
||||
endpoint = '/'.join(parts[:-3])
|
||||
return (image_id, endpoint)
|
||||
|
||||
|
||||
def generate_identity_headers(context, status='Confirmed'):
|
||||
return {
|
||||
'X-Auth-Token': getattr(context, 'auth_token', None),
|
||||
'X-User-Id': getattr(context, 'user_id', None),
|
||||
'X-Tenant-Id': getattr(context, 'project_id', None),
|
||||
'X-Roles': ','.join(getattr(context, 'roles', [])),
|
||||
'X-Identity-Status': status,
|
||||
}
|
||||
|
||||
|
||||
def get_api_servers(context):
|
||||
"""Shuffle a list of service endpoints and return an iterator that will
|
||||
cycle through the list, looping around to the beginning if necessary.
|
||||
"""
|
||||
# NOTE(efried): utils.get_ksa_adapter().get_endpoint() is the preferred
|
||||
# mechanism for endpoint discovery. Only use `api_servers` if you really
|
||||
# need to shuffle multiple endpoints.
|
||||
if CONF.glance.api_servers:
|
||||
api_servers = CONF.glance.api_servers
|
||||
random.shuffle(api_servers)
|
||||
else:
|
||||
sess, auth = _session_and_auth(context)
|
||||
ksa_adap = utils.get_ksa_adapter(
|
||||
cyborg.conf.glance.DEFAULT_SERVICE_TYPE,
|
||||
ksa_auth=auth, ksa_session=sess,
|
||||
min_version='2.0', max_version='2.latest')
|
||||
endpoint = utils.get_endpoint(ksa_adap)
|
||||
if endpoint:
|
||||
# NOTE(mriedem): Due to python-glanceclient bug 1707995 we have
|
||||
# to massage the endpoint URL otherwise it won't work properly.
|
||||
# We can't use glanceclient.common.utils.strip_version because
|
||||
# of bug 1748009.
|
||||
endpoint = re.sub(r'/v\d+(\.\d+)?/?$', '/', endpoint)
|
||||
api_servers = [endpoint]
|
||||
|
||||
return itertools.cycle(api_servers)
|
||||
|
||||
|
||||
class GlanceClientWrapper(object):
|
||||
"""Glance client wrapper class that implements retries."""
|
||||
|
||||
def __init__(self, context=None, endpoint=None):
|
||||
version = 2
|
||||
if endpoint is not None:
|
||||
self.client = self._create_static_client(context,
|
||||
endpoint,
|
||||
version)
|
||||
else:
|
||||
self.client = None
|
||||
self.api_servers = None
|
||||
|
||||
def _create_static_client(self, context, endpoint, version):
|
||||
"""Create a client that we'll use for every call."""
|
||||
self.api_server = str(endpoint)
|
||||
return _glanceclient_from_endpoint(context, endpoint, version)
|
||||
|
||||
def _create_onetime_client(self, context, version):
|
||||
"""Create a client that will be used for one call."""
|
||||
if self.api_servers is None:
|
||||
self.api_servers = get_api_servers(context)
|
||||
self.api_server = next(self.api_servers)
|
||||
return _glanceclient_from_endpoint(context, self.api_server, version)
|
||||
|
||||
def call(self, context, version, method, *args, **kwargs):
|
||||
"""Call a glance client method. If we get a connection error,
|
||||
retry the request according to CONF.glance.num_retries.
|
||||
"""
|
||||
retry_excs = (glanceclient.exc.ServiceUnavailable,
|
||||
glanceclient.exc.InvalidEndpoint,
|
||||
glanceclient.exc.CommunicationError)
|
||||
num_attempts = 1 + CONF.glance.num_retries
|
||||
|
||||
for attempt in range(1, num_attempts + 1):
|
||||
client = self.client or self._create_onetime_client(context,
|
||||
version)
|
||||
try:
|
||||
controller = getattr(client,
|
||||
kwargs.pop('controller', 'images'))
|
||||
result = getattr(controller, method)(*args, **kwargs)
|
||||
if inspect.isgenerator(result):
|
||||
# Convert generator results to a list, so that we can
|
||||
# catch any potential exceptions now and retry the call.
|
||||
return list(result)
|
||||
return result
|
||||
except retry_excs as e:
|
||||
if attempt < num_attempts:
|
||||
extra = "retrying"
|
||||
else:
|
||||
extra = 'done trying'
|
||||
|
||||
LOG.exception("Error contacting glance server "
|
||||
"'%(server)s' for '%(method)s', "
|
||||
"%(extra)s.",
|
||||
{'server': self.api_server,
|
||||
'method': method, 'extra': extra})
|
||||
if attempt == num_attempts:
|
||||
raise exception.GlanceConnectionFailed(
|
||||
server=str(self.api_server), reason=six.text_type(e))
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
class GlanceImageServiceV2(object):
|
||||
"""Provides storage and retrieval of disk image objects within Glance."""
|
||||
|
||||
def __init__(self, client=None):
|
||||
self._client = client or GlanceClientWrapper()
|
||||
# NOTE(jbresnah) build the table of download handlers at the beginning
|
||||
# so that operators can catch errors at load time rather than whenever
|
||||
# a user attempts to use a module. Note this cannot be done in glance
|
||||
# space when this python module is loaded because the download module
|
||||
# may require configuration options to be parsed.
|
||||
self._download_handlers = {}
|
||||
download_modules = image_xfers.load_transfer_modules()
|
||||
|
||||
for scheme, mod in download_modules.items():
|
||||
if scheme not in CONF.glance.allowed_direct_url_schemes:
|
||||
continue
|
||||
|
||||
try:
|
||||
self._download_handlers[scheme] = mod.get_download_handler()
|
||||
except Exception as ex:
|
||||
LOG.error('When loading the module %(module_str)s the '
|
||||
'following error occurred: %(ex)s',
|
||||
{'module_str': str(mod), 'ex': ex})
|
||||
|
||||
@staticmethod
|
||||
def _safe_fsync(fh):
|
||||
"""Performs os.fsync on a filehandle only if it is supported.
|
||||
|
||||
fsync on a pipe, FIFO, or socket raises OSError with EINVAL. This
|
||||
method discovers whether the target filehandle is one of these types
|
||||
and only performs fsync if it isn't.
|
||||
|
||||
:param fh: Open filehandle (not a path or fileno) to maybe fsync.
|
||||
"""
|
||||
fileno = fh.fileno()
|
||||
mode = os.fstat(fileno).st_mode
|
||||
# A pipe answers True to S_ISFIFO
|
||||
if not any(check(mode) for check in (stat.S_ISFIFO, stat.S_ISSOCK)):
|
||||
os.fsync(fileno)
|
||||
|
||||
def download(self, context, image_id, data=None, dst_path=None):
|
||||
"""Calls out to Glance for data and writes data."""
|
||||
if CONF.glance.allowed_direct_url_schemes and dst_path is not None:
|
||||
image = self.show(context, image_id, include_locations=True)
|
||||
for entry in image.get('locations', []):
|
||||
loc_url = entry['url']
|
||||
loc_meta = entry['metadata']
|
||||
o = urlparse.urlparse(loc_url)
|
||||
xfer_mod = self._get_transfer_module(o.scheme)
|
||||
if xfer_mod:
|
||||
try:
|
||||
xfer_mod.download(context, o, dst_path, loc_meta)
|
||||
LOG.info("Successfully transferred using %s", o.scheme)
|
||||
return
|
||||
except Exception:
|
||||
LOG.exception("Download image error")
|
||||
|
||||
try:
|
||||
image_chunks = self._client.call(context, 2, 'data', image_id)
|
||||
except Exception:
|
||||
_reraise_translated_image_exception(image_id)
|
||||
|
||||
if image_chunks.wrapped is None:
|
||||
# None is a valid return value, but there's nothing we can do with
|
||||
# a image with no associated data
|
||||
raise exception.ImageUnacceptable(image_id=image_id,
|
||||
reason='Image has no \
|
||||
associated data')
|
||||
|
||||
# Retrieve properties for verification of Glance image signature
|
||||
verifier = None
|
||||
if CONF.glance.verify_glance_signatures:
|
||||
image_meta_dict = self.show(context, image_id,
|
||||
include_locations=False)
|
||||
image_meta = objects.ImageMeta.from_dict(image_meta_dict)
|
||||
img_signature = image_meta.properties.get('img_signature')
|
||||
img_sig_hash_method = image_meta.properties.get(
|
||||
'img_signature_hash_method'
|
||||
)
|
||||
img_sig_cert_uuid = image_meta.properties.get(
|
||||
'img_signature_certificate_uuid'
|
||||
)
|
||||
img_sig_key_type = image_meta.properties.get(
|
||||
'img_signature_key_type'
|
||||
)
|
||||
try:
|
||||
verifier = signature_utils.get_verifier(
|
||||
context=context,
|
||||
img_signature_certificate_uuid=img_sig_cert_uuid,
|
||||
img_signature_hash_method=img_sig_hash_method,
|
||||
img_signature=img_signature,
|
||||
img_signature_key_type=img_sig_key_type,
|
||||
)
|
||||
except cursive_exception.SignatureVerificationError:
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.error('Image signature verification failed '
|
||||
'for image: %s', image_id)
|
||||
|
||||
close_file = False
|
||||
if data is None and dst_path:
|
||||
data = open(dst_path, 'wb')
|
||||
close_file = True
|
||||
|
||||
if data is None:
|
||||
|
||||
# Perform image signature verification
|
||||
if verifier:
|
||||
try:
|
||||
for chunk in image_chunks:
|
||||
verifier.update(chunk)
|
||||
verifier.verify()
|
||||
|
||||
LOG.info('Image signature verification succeeded '
|
||||
'for image: %s', image_id)
|
||||
|
||||
except cryptography.exceptions.InvalidSignature:
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.error('Image signature verification failed '
|
||||
'for image: %s', image_id)
|
||||
return image_chunks
|
||||
else:
|
||||
try:
|
||||
for chunk in image_chunks:
|
||||
if verifier:
|
||||
verifier.update(chunk)
|
||||
data.write(chunk)
|
||||
if verifier:
|
||||
verifier.verify()
|
||||
LOG.info('Image signature verification succeeded '
|
||||
'for image %s', image_id)
|
||||
except cryptography.exceptions.InvalidSignature:
|
||||
data.truncate(0)
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.error('Image signature verification failed '
|
||||
'for image: %s', image_id)
|
||||
except Exception as ex:
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.error("Error writing to %(path)s: %(exception)s",
|
||||
{'path': dst_path, 'exception': ex})
|
||||
finally:
|
||||
if close_file:
|
||||
# Ensure that the data is pushed all the way down to
|
||||
# persistent storage. This ensures that in the event of a
|
||||
# subsequent host crash we don't have running instances
|
||||
# using a corrupt backing file.
|
||||
data.flush()
|
||||
self._safe_fsync(data)
|
||||
data.close()
|
||||
|
||||
|
||||
def _extract_query_params(params):
|
||||
_params = {}
|
||||
accepted_params = ('filters', 'marker', 'limit',
|
||||
'page_size', 'sort_key', 'sort_dir')
|
||||
for param in accepted_params:
|
||||
if params.get(param):
|
||||
_params[param] = params.get(param)
|
||||
|
||||
# ensure filters is a dict
|
||||
_params.setdefault('filters', {})
|
||||
# NOTE(vish): don't filter out private images
|
||||
_params['filters'].setdefault('is_public', 'none')
|
||||
|
||||
return _params
|
||||
|
||||
|
||||
def _extract_query_params_v2(params):
|
||||
_params = {}
|
||||
accepted_params = ('filters', 'marker', 'limit',
|
||||
'page_size', 'sort_key', 'sort_dir')
|
||||
for param in accepted_params:
|
||||
if params.get(param):
|
||||
_params[param] = params.get(param)
|
||||
|
||||
# ensure filters is a dict
|
||||
_params.setdefault('filters', {})
|
||||
# NOTE(vish): don't filter out private images
|
||||
_params['filters'].setdefault('is_public', 'none')
|
||||
|
||||
# adopt filters to be accepted by glance v2 api
|
||||
filters = _params['filters']
|
||||
new_filters = {}
|
||||
|
||||
for filter_ in filters:
|
||||
# remove 'property-' prefix from filters by custom properties
|
||||
if filter_.startswith('property-'):
|
||||
new_filters[filter_.lstrip('property-')] = filters[filter_]
|
||||
elif filter_ == 'changes-since':
|
||||
# convert old 'changes-since' into new 'updated_at' filter
|
||||
updated_at = 'gte:' + filters['changes-since']
|
||||
new_filters['updated_at'] = updated_at
|
||||
elif filter_ == 'is_public':
|
||||
# convert old 'is_public' flag into 'visibility' filter
|
||||
# omit the filter if is_public is None
|
||||
is_public = filters['is_public']
|
||||
if is_public.lower() in ('true', '1'):
|
||||
new_filters['visibility'] = 'public'
|
||||
elif is_public.lower() in ('false', '0'):
|
||||
new_filters['visibility'] = 'private'
|
||||
else:
|
||||
new_filters[filter_] = filters[filter_]
|
||||
|
||||
_params['filters'] = new_filters
|
||||
|
||||
return _params
|
||||
|
||||
|
||||
def _is_image_available(context, image):
|
||||
"""Check image availability.
|
||||
|
||||
This check is needed in case cyborg and Glance are deployed
|
||||
without authentication turned on.
|
||||
"""
|
||||
# The presence of an auth token implies this is an authenticated
|
||||
# request and we need not handle the noauth use-case.
|
||||
if hasattr(context, 'auth_token') and context.auth_token:
|
||||
return True
|
||||
|
||||
def _is_image_public(image):
|
||||
# NOTE(jaypipes) V2 Glance API replaced the is_public attribute
|
||||
# with a visibility attribute. We do this here to prevent the
|
||||
# glanceclient for a V2 image model from throwing an
|
||||
# exception from warlock when trying to access an is_public
|
||||
# attribute.
|
||||
if hasattr(image, 'visibility'):
|
||||
return str(image.visibility).lower() == 'public'
|
||||
else:
|
||||
return image.is_public
|
||||
|
||||
if context.is_admin or _is_image_public(image):
|
||||
return True
|
||||
|
||||
properties = image.properties
|
||||
|
||||
if context.project_id and ('owner_id' in properties):
|
||||
return str(properties['owner_id']) == str(context.project_id)
|
||||
|
||||
if context.project_id and ('project_id' in properties):
|
||||
return str(properties['project_id']) == str(context.project_id)
|
||||
|
||||
try:
|
||||
user_id = properties['user_id']
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
return str(user_id) == str(context.user_id)
|
||||
|
||||
|
||||
def _translate_to_glance(image_meta):
|
||||
image_meta = _convert_to_string(image_meta)
|
||||
image_meta = _remove_read_only(image_meta)
|
||||
image_meta = _convert_to_v2(image_meta)
|
||||
return image_meta
|
||||
|
||||
|
||||
def _convert_to_v2(image_meta):
|
||||
output = {}
|
||||
for name, value in image_meta.items():
|
||||
if name == 'properties':
|
||||
for prop_name, prop_value in value.items():
|
||||
# if allow_additional_image_properties is disabled we can't
|
||||
# define kernel_id and ramdisk_id as None, so we have to omit
|
||||
# these properties if they are not set.
|
||||
if prop_name in ('kernel_id', 'ramdisk_id') and \
|
||||
prop_value is not None and \
|
||||
prop_value.strip().lower() in ('none', ''):
|
||||
continue
|
||||
# in glance only string and None property values are allowed,
|
||||
# v1 client accepts any values and converts them to string,
|
||||
# v2 doesn't - so we have to take care of it.
|
||||
elif prop_value is None or isinstance(
|
||||
prop_value, six.string_types):
|
||||
output[prop_name] = prop_value
|
||||
else:
|
||||
output[prop_name] = str(prop_value)
|
||||
|
||||
elif name in ('min_ram', 'min_disk'):
|
||||
output[name] = int(value)
|
||||
elif name == 'is_public':
|
||||
output['visibility'] = 'public' if value else 'private'
|
||||
elif name in ('size', 'deleted'):
|
||||
continue
|
||||
else:
|
||||
output[name] = value
|
||||
return output
|
||||
|
||||
|
||||
def _translate_from_glance(image, include_locations=False):
|
||||
image_meta = _extract_attributes_v2(
|
||||
image, include_locations=include_locations)
|
||||
|
||||
image_meta = _convert_timestamps_to_datetimes(image_meta)
|
||||
image_meta = _convert_from_string(image_meta)
|
||||
return image_meta
|
||||
|
||||
|
||||
def _convert_timestamps_to_datetimes(image_meta):
|
||||
"""Returns image with timestamp fields converted to datetime objects."""
|
||||
for attr in ['created_at', 'updated_at', 'deleted_at']:
|
||||
if image_meta.get(attr):
|
||||
image_meta[attr] = timeutils.parse_isotime(image_meta[attr])
|
||||
return image_meta
|
||||
|
||||
|
||||
# NOTE(bcwaldon): used to store non-string data in glance metadata
|
||||
def _json_loads(properties, attr):
|
||||
prop = properties[attr]
|
||||
if isinstance(prop, six.string_types):
|
||||
properties[attr] = jsonutils.loads(prop)
|
||||
|
||||
|
||||
def _json_dumps(properties, attr):
|
||||
prop = properties[attr]
|
||||
if not isinstance(prop, six.string_types):
|
||||
properties[attr] = jsonutils.dumps(prop)
|
||||
|
||||
|
||||
_CONVERT_PROPS = ('block_device_mapping', 'mappings')
|
||||
|
||||
|
||||
def _convert(method, metadata):
|
||||
metadata = copy.deepcopy(metadata)
|
||||
properties = metadata.get('properties')
|
||||
if properties:
|
||||
for attr in _CONVERT_PROPS:
|
||||
if attr in properties:
|
||||
method(properties, attr)
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def _convert_from_string(metadata):
|
||||
return _convert(_json_loads, metadata)
|
||||
|
||||
|
||||
def _convert_to_string(metadata):
|
||||
return _convert(_json_dumps, metadata)
|
||||
|
||||
|
||||
def _extract_attributes(image, include_locations=False):
|
||||
# TODO(mfedosin): Remove this function once we move to glance V2
|
||||
# completely.
|
||||
# NOTE(hdd): If a key is not found, base.Resource.__getattr__() may perform
|
||||
# a get(), resulting in a useless request back to glance. This list is
|
||||
# therefore sorted, with dependent attributes as the end
|
||||
# 'deleted_at' depends on 'deleted'
|
||||
# 'checksum' depends on 'status' == 'active'
|
||||
IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner',
|
||||
'container_format', 'status', 'id',
|
||||
'name', 'created_at', 'updated_at',
|
||||
'deleted', 'deleted_at', 'checksum',
|
||||
'min_disk', 'min_ram', 'is_public',
|
||||
'direct_url', 'locations']
|
||||
|
||||
queued = getattr(image, 'status') == 'queued'
|
||||
queued_exclude_attrs = ['disk_format', 'container_format']
|
||||
include_locations_attrs = ['direct_url', 'locations']
|
||||
output = {}
|
||||
|
||||
for attr in IMAGE_ATTRIBUTES:
|
||||
if attr == 'deleted_at' and not output['deleted']:
|
||||
output[attr] = None
|
||||
elif attr == 'checksum' and output['status'] != 'active':
|
||||
output[attr] = None
|
||||
# image may not have 'name' attr
|
||||
elif attr == 'name':
|
||||
output[attr] = getattr(image, attr, None)
|
||||
# NOTE(liusheng): queued image may not have these attributes and 'name'
|
||||
elif queued and attr in queued_exclude_attrs:
|
||||
output[attr] = getattr(image, attr, None)
|
||||
# NOTE(mriedem): Only get location attrs if including locations.
|
||||
elif attr in include_locations_attrs:
|
||||
if include_locations:
|
||||
output[attr] = getattr(image, attr, None)
|
||||
# NOTE(mdorman): 'size' attribute must not be 'None', so use 0 instead
|
||||
elif attr == 'size':
|
||||
# NOTE(mriedem): A snapshot image may not have the size attribute
|
||||
# set so default to 0.
|
||||
output[attr] = getattr(image, attr, 0) or 0
|
||||
else:
|
||||
# NOTE(xarses): Anything that is caught with the default value
|
||||
# will result in an additional lookup to glance for said attr.
|
||||
# Notable attributes that could have this issue:
|
||||
# disk_format, container_format, name, deleted, checksum
|
||||
output[attr] = getattr(image, attr, None)
|
||||
|
||||
output['properties'] = getattr(image, 'properties', {})
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def _extract_attributes_v2(image, include_locations=False):
|
||||
include_locations_attrs = ['direct_url', 'locations']
|
||||
omit_attrs = ['self', 'schema', 'protected', 'virtual_size', 'file',
|
||||
'tags']
|
||||
raw_schema = image.schema
|
||||
schema = schemas.Schema(raw_schema)
|
||||
output = {'properties': {}, 'deleted': False, 'deleted_at': None,
|
||||
'disk_format': None, 'container_format': None, 'name': None,
|
||||
'checksum': None}
|
||||
for name, value in image.items():
|
||||
if (name in omit_attrs
|
||||
or name in include_locations_attrs and not include_locations):
|
||||
continue
|
||||
elif name == 'visibility':
|
||||
output['is_public'] = value == 'public'
|
||||
elif name == 'size' and value is None:
|
||||
output['size'] = 0
|
||||
elif schema.is_base_property(name):
|
||||
output[name] = value
|
||||
else:
|
||||
output['properties'][name] = value
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def _remove_read_only(image_meta):
|
||||
IMAGE_ATTRIBUTES = ['status', 'updated_at', 'created_at', 'deleted_at']
|
||||
output = copy.deepcopy(image_meta)
|
||||
for attr in IMAGE_ATTRIBUTES:
|
||||
if attr in output:
|
||||
del output[attr]
|
||||
return output
|
||||
|
||||
|
||||
def _reraise_translated_image_exception(image_id):
|
||||
"""Transform the exception for the image but keep its traceback intact."""
|
||||
exc_type, exc_value, exc_trace = sys.exc_info()
|
||||
new_exc = _translate_image_exception(image_id, exc_value)
|
||||
six.reraise(type(new_exc), new_exc, exc_trace)
|
||||
|
||||
|
||||
def _reraise_translated_exception():
|
||||
"""Transform the exception but keep its traceback intact."""
|
||||
exc_type, exc_value, exc_trace = sys.exc_info()
|
||||
new_exc = _translate_plain_exception(exc_value)
|
||||
six.reraise(type(new_exc), new_exc, exc_trace)
|
||||
|
||||
|
||||
def _translate_image_exception(image_id, exc_value):
|
||||
if isinstance(exc_value, (glanceclient.exc.Forbidden,
|
||||
glanceclient.exc.Unauthorized)):
|
||||
return exception.ImageNotAuthorized(image_id=image_id)
|
||||
if isinstance(exc_value, glanceclient.exc.NotFound):
|
||||
return exception.ImageNotFound(image_id=image_id)
|
||||
if isinstance(exc_value, glanceclient.exc.BadRequest):
|
||||
return exception.ImageBadRequest(image_id=image_id,
|
||||
response=six.text_type(exc_value))
|
||||
return exc_value
|
||||
|
||||
|
||||
def _translate_plain_exception(exc_value):
|
||||
if isinstance(exc_value, (glanceclient.exc.Forbidden,
|
||||
glanceclient.exc.Unauthorized)):
|
||||
return exception.Forbidden(six.text_type(exc_value))
|
||||
if isinstance(exc_value, glanceclient.exc.NotFound):
|
||||
return exception.NotFound(six.text_type(exc_value))
|
||||
if isinstance(exc_value, glanceclient.exc.BadRequest):
|
||||
return exception.Invalid(six.text_type(exc_value))
|
||||
return exc_value
|
||||
|
||||
|
||||
def get_remote_image_service(context, image_href):
|
||||
"""Create an image_service and parse the id from the given image_href.
|
||||
|
||||
The image_href param can be an href of the form
|
||||
'http://example.com:9292/v1/images/b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3',
|
||||
or just an id such as 'b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3'. If the
|
||||
image_href is a standalone id, then the default image service is returned.
|
||||
|
||||
:param image_href: href that describes the location of an image
|
||||
:returns: a tuple of the form (image_service, image_id)
|
||||
|
||||
"""
|
||||
# NOTE(bcwaldon): If image_href doesn't look like a URI, assume its a
|
||||
# standalone image ID
|
||||
if '/' not in str(image_href):
|
||||
image_service = get_default_image_service()
|
||||
return image_service, image_href
|
||||
|
||||
try:
|
||||
(image_id, endpoint) = _endpoint_from_image_ref(image_href)
|
||||
glance_client = GlanceClientWrapper(context=context,
|
||||
endpoint=endpoint)
|
||||
except ValueError:
|
||||
raise exception.InvalidImageRef(image_href=image_href)
|
||||
|
||||
image_service = GlanceImageServiceV2(client=glance_client)
|
||||
return image_service, image_id
|
||||
|
||||
|
||||
def get_default_image_service():
|
||||
return GlanceImageServiceV2()
|
||||
|
||||
|
||||
class UpdateGlanceImage(object):
|
||||
def __init__(self, context, image_id, metadata, stream):
|
||||
self.context = context
|
||||
self.image_id = image_id
|
||||
self.metadata = metadata
|
||||
self.image_stream = stream
|
||||
|
||||
def start(self):
|
||||
image_service, image_id = (
|
||||
get_remote_image_service(self.context, self.image_id))
|
||||
image_service.update(self.context, image_id, self.metadata,
|
||||
self.image_stream, purge_props=False)
|
||||
54
cyborg/service_auth.py
Normal file
54
cyborg/service_auth.py
Normal file
@ -0,0 +1,54 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
from keystoneauth1 import loading as ks_loading
|
||||
from keystoneauth1 import service_token
|
||||
from oslo_log import log as logging
|
||||
|
||||
import cyborg.conf
|
||||
|
||||
|
||||
CONF = cyborg.conf.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
_SERVICE_AUTH = None
|
||||
|
||||
|
||||
def reset_globals():
|
||||
"""For async unit test consistency."""
|
||||
global _SERVICE_AUTH
|
||||
_SERVICE_AUTH = None
|
||||
|
||||
|
||||
def get_auth_plugin(context):
|
||||
user_auth = context.get_auth_plugin()
|
||||
|
||||
if CONF.service_user.send_service_user_token:
|
||||
global _SERVICE_AUTH
|
||||
if not _SERVICE_AUTH:
|
||||
_SERVICE_AUTH = ks_loading.\
|
||||
load_auth_from_conf_options(CONF,
|
||||
group=cyborg.
|
||||
conf.service_token.
|
||||
SERVICE_USER_GROUP)
|
||||
if _SERVICE_AUTH is None:
|
||||
# This indicates a misconfiguration so log a warning and
|
||||
# return the user_auth.
|
||||
LOG.warning('Unable to load auth from [service_user] '
|
||||
'configuration. Ensure "auth_type" is set.')
|
||||
return user_auth
|
||||
return service_token.\
|
||||
ServiceTokenAuthWrapper(user_auth=user_auth,
|
||||
service_auth=_SERVICE_AUTH)
|
||||
|
||||
return user_auth
|
||||
Loading…
Reference in New Issue
Block a user