Added Glance Client for Image downloading

According to sean mooney's comments in
https://review.openstack.org/#/c/596507/

I am splitting into 2 patches

For now, please add following session in cyborg.conf
[glance]
project_domain_name = Default
project_name = service
user_domain_name = Default
password = {your_password}
username = placement
auth_url = http://{host_ip}/identity
auth_type = password
auth_section = keystone_authtoken
api_servers = http://{host_ip}/image

Change-Id: I3a0249c7ac8ba5fd3c20ae6402dcc2f9e9bcfe95

cyborg/common/exception.py

@@ -305,3 +305,25 @@ class QuotaResourceUnknown(QuotaNotFound):
 
 class InvalidReservationExpiration(Invalid):
     message = _("Invalid reservation expiration %(expire)s.")
+
+
+class GlanceConnectionFailed(CyborgException):
+    msg_fmt = _("Connection to glance host %(server)s failed: "
+                "%(reason)s")
+
+
+class ImageUnacceptable(Invalid):
+    msg_fmt = _("Image %(image_id)s is unacceptable: %(reason)s")
+
+
+class ImageNotAuthorized(CyborgException):
+    msg_fmt = _("Not authorized for image %(image_id)s.")
+
+
+class ImageNotFound(NotFound):
+    msg_fmt = _("Image %(image_id)s could not be found.")
+
+
+class ImageBadRequest(Invalid):
+    msg_fmt = _("Request of image %(image_id)s got BadRequest response: "
+                "%(response)s")

cyborg/common/rpc.py

@@ -14,7 +14,7 @@
 #    under the License.
 
 from oslo_config import cfg
-from oslo_context import context as cyborg_context
+from cyborg import context as cyborg_context
 import oslo_messaging as messaging
 from oslo_messaging.rpc import dispatcher
 

cyborg/common/service.py

@@ -14,7 +14,7 @@
 #    under the License.
 
 from oslo_concurrency import processutils
-from oslo_context import context
+from cyborg import context
 from oslo_log import log
 import oslo_messaging as messaging
 from oslo_service import service

cyborg/common/utils.py

@@ -108,3 +108,57 @@ def get_ksa_adapter(service_type, ksa_auth=None, ksa_session=None,
     return ks_loading.load_adapter_from_conf_options(
         CONF, confgrp, session=ksa_session, auth=ksa_auth,
         min_version=min_version, max_version=max_version)
+
+
+def get_endpoint(ksa_adapter):
+    """Get the endpoint URL represented by a keystoneauth1 Adapter.
+
+    This method is equivalent to what
+
+        ksa_adapter.get_endpoint()
+
+    should do, if it weren't for a panoply of bugs.
+
+    :param ksa_adapter: keystoneauth1.adapter.Adapter, appropriately set up
+                        with an endpoint_override; or service_type, interface
+                        (list) and auth/service_catalog.
+    :return: String endpoint URL.
+    :raise EndpointNotFound: If endpoint discovery fails.
+    """
+    # TODO(efried): This will be unnecessary once bug #1707993 is fixed.
+    # (At least for the non-image case, until 1707995 is fixed.)
+    if ksa_adapter.endpoint_override:
+        return ksa_adapter.endpoint_override
+    # TODO(efried): Remove this once bug #1707995 is fixed.
+    if ksa_adapter.service_type == 'image':
+        try:
+            # LOG.warning(ksa_adapter.__dict__)
+            return ksa_adapter.get_endpoint_data().catalog_url
+        except AttributeError:
+            # ksa_adapter.auth is a _ContextAuthPlugin, which doesn't have
+            # get_endpoint_data.  Fall through to using get_endpoint().
+            pass
+    # TODO(efried): The remainder of this method reduces to
+    # TODO(efried):     return ksa_adapter.get_endpoint()
+    # TODO(efried): once bug #1709118 is fixed.
+    # NOTE(efried): Id9bd19cca68206fc64d23b0eaa95aa3e5b01b676 may also do the
+    #               trick, once it's in a ksa release.
+    # The EndpointNotFound exception happens when _ContextAuthPlugin is in play
+    # because its get_endpoint() method isn't yet set up to handle interface as
+    # a list.  (It could also happen with a real auth if the endpoint isn't
+    # there; but that's covered below.)
+    try:
+        return ksa_adapter.get_endpoint()
+    except ks_exc.EndpointNotFound:
+        pass
+
+    interfaces = list(ksa_adapter.interface)
+    for interface in interfaces:
+        ksa_adapter.interface = interface
+        try:
+            return ksa_adapter.get_endpoint()
+        except ks_exc.EndpointNotFound:
+            pass
+    raise ks_exc.EndpointNotFound(
+        "Could not find requested endpoint for any of the following "
+        "interfaces: %s" % interfaces)

cyborg/conf/__init__.py

@@ -18,7 +18,9 @@ from oslo_config import cfg
 from cyborg.conf import api
 from cyborg.conf import database
 from cyborg.conf import default
-
+from cyborg.conf import service_token
+from cyborg.conf import glance
+from cyborg.conf import keystone
 
 CONF = cfg.CONF
 
@@ -26,3 +28,6 @@ api.register_opts(CONF)
 database.register_opts(CONF)
 default.register_opts(CONF)
 default.register_placement_opts(CONF)
+service_token.register_opts(CONF)
+glance.register_opts(CONF)
+keystone.register_opts(CONF)

cyborg/conf/glance.py

@@ -0,0 +1,171 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from keystoneauth1 import loading as ks_loading
+from oslo_config import cfg
+
+from cyborg.conf import utils as confutils
+
+
+DEFAULT_SERVICE_TYPE = 'image'
+
+glance_group = cfg.OptGroup(
+    'glance',
+    title='Glance Options',
+    help='Configuration options for the Image service')
+
+glance_opts = [
+    # NOTE(sdague/efried): there is intentionally no default here. This
+    # requires configuration if ksa adapter config is not used.
+    cfg.ListOpt('api_servers',
+                help="""
+List of glance api servers endpoints available to cyborg.
+
+https is used for ssl-based glance api servers.
+
+NOTE: The preferred mechanism for endpoint discovery is via keystoneauth1
+loading options. Only use api_servers if you need multiple endpoints and are
+unable to use a load balancer for some reason.
+
+Possible values:
+
+* A list of any fully qualified url of the form "scheme://hostname:port[/path]"
+  (i.e. "http://10.0.1.0:9292" or "https://my.glance.server/image").
+"""),
+    cfg.IntOpt('num_retries',
+               default=0,
+               min=0,
+               help="""
+Enable glance operation retries.
+
+Specifies the number of retries when uploading / downloading
+an image to / from glance. 0 means no retries.
+"""),
+    cfg.ListOpt('allowed_direct_url_schemes',
+                default=[],
+                deprecated_for_removal=True,
+                deprecated_since='17.0.0',
+                deprecated_reason="""
+This was originally added for the 'cyborg.image.download.file' FileTransfer
+extension which was removed in the 16.0.0 Pike release. The
+'cyborg.image.download.modules' extension point is not maintained
+and there is no indication of its use in production clouds.
+""",
+                help="""
+List of url schemes that can be directly accessed.
+
+This option specifies a list of url schemes that can be downloaded
+directly via the direct_url. This direct_URL can be fetched from
+Image metadata which can be used by cyborg to get the
+image more efficiently. cyborg-compute could benefit from this by
+invoking a copy when it has access to the same file system as glance.
+
+Possible values:
+
+* [file], Empty list (default)
+"""),
+    cfg.BoolOpt('verify_glance_signatures',
+                default=False,
+                help="""
+Enable image signature verification.
+
+cyborg uses the image signature metadata from glance and verifies the signature
+of a signed image while downloading that image. If the image signature cannot
+be verified or if the image signature metadata is either incomplete or
+unavailable, then cyborg will not boot the image and instead will place the
+instance into an error state. This provides end users with stronger assurances
+of the integrity of the image data they are using to create servers.
+
+Related options:
+
+* The options in the `key_manager` group, as the key_manager is used
+  for the signature validation.
+* Both enable_certificate_validation and default_trusted_certificate_ids
+  below depend on this option being enabled.
+"""),
+    cfg.BoolOpt('enable_certificate_validation',
+                default=False,
+                deprecated_for_removal=True,
+                deprecated_since='16.0.0',
+                deprecated_reason="""
+This option is intended to ease the transition for deployments leveraging
+image signature verification. The intended state long-term is for signature
+verification and certificate validation to always happen together.
+""",
+                help="""
+Enable certificate validation for image signature verification.
+
+During image signature verification cyborg will first verify the validity of
+the image's signing certificate using the set of trusted certificates
+associated with the instance. If certificate validation fails, signature
+verification will not be performed and the instance will be placed into an
+error state. This provides end users with stronger assurances that the image
+data is unmodified and trustworthy. If left disabled, image signature
+verification can still occur but the end user will not have any assurance that
+the signing certificate used to generate the image signature is still
+trustworthy.
+
+Related options:
+
+* This option only takes effect if verify_glance_signatures is enabled.
+* The value of default_trusted_certificate_ids may be used when this option
+  is enabled.
+"""),
+    cfg.ListOpt('default_trusted_certificate_ids',
+                default=[],
+                help="""
+List of certificate IDs for certificates that should be trusted.
+
+May be used as a default list of trusted certificate IDs for certificate
+validation. The value of this option will be ignored if the user provides a
+list of trusted certificate IDs with an instance API request. The value of
+this option will be persisted with the instance data if signature verification
+and certificate validation are enabled and if the user did not provide an
+alternative list. If left empty when certificate validation is enabled the
+user must provide a list of trusted certificate IDs otherwise certificate
+validation will fail.
+
+Related options:
+
+* The value of this option may be used if both verify_glance_signatures and
+  enable_certificate_validation are enabled.
+"""),
+    cfg.BoolOpt('debug',
+                default=False,
+                help='Enable or disable debug logging with glanceclient.')
+]
+
+deprecated_ksa_opts = {
+    'insecure': [cfg.DeprecatedOpt('api_insecure', group=glance_group.name)],
+    'cafile': [cfg.DeprecatedOpt('ca_file', group="ssl")],
+    'certfile': [cfg.DeprecatedOpt('cert_file', group="ssl")],
+    'keyfile': [cfg.DeprecatedOpt('key_file', group="ssl")],
+}
+
+
+def register_opts(conf):
+    conf.register_group(glance_group)
+    conf.register_opts(glance_opts, group=glance_group)
+
+    confutils.register_ksa_opts(
+        conf, glance_group, DEFAULT_SERVICE_TYPE, include_auth=False,
+        deprecated_opts=deprecated_ksa_opts)
+
+
+def list_opts():
+    return {glance_group: (
+        glance_opts +
+        ks_loading.get_session_conf_options() +
+        confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE,
+                                       deprecated_opts=deprecated_ksa_opts))}

cyborg/conf/keystone.py

@@ -0,0 +1,40 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from keystoneauth1 import loading as ks_loading
+from oslo_config import cfg
+
+from cyborg.conf import utils as confutils
+
+
+DEFAULT_SERVICE_TYPE = 'identity'
+
+keystone_group = cfg.OptGroup(
+    'keystone',
+    title='Keystone Options',
+    help='Configuration options for the identity service')
+
+
+def register_opts(conf):
+    conf.register_group(keystone_group)
+    confutils.register_ksa_opts(conf, keystone_group.name,
+                                DEFAULT_SERVICE_TYPE, include_auth=False)
+
+
+def list_opts():
+    return {
+        keystone_group: (
+            ks_loading.get_session_conf_options() +
+            confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE)
+        )
+    }

cyborg/conf/service_token.py

@@ -0,0 +1,54 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from keystoneauth1 import loading as ks_loading
+from oslo_config import cfg
+
+SERVICE_USER_GROUP = 'service_user'
+
+service_user = cfg.OptGroup(
+    SERVICE_USER_GROUP,
+    title='Service token authentication type options',
+    help="""
+Configuration options for service to service authentication using a service
+token. These options allow sending a service token along with the user's token
+when contacting external REST APIs.
+"""
+)
+
+service_user_opts = [
+    cfg.BoolOpt('send_service_user_token',
+                default=False,
+                help="""
+When True, if sending a user token to a REST API, also send a service token.
+"""),
+]
+
+
+def register_opts(conf):
+    conf.register_group(service_user)
+    conf.register_opts(service_user_opts, group=service_user)
+
+    ks_loading.register_session_conf_options(conf, SERVICE_USER_GROUP)
+    ks_loading.register_auth_conf_options(conf, SERVICE_USER_GROUP)
+
+
+def list_opts():
+    return {
+        service_user: (
+            service_user_opts +
+            ks_loading.get_session_conf_options() +
+            ks_loading.get_auth_common_conf_options() +
+            ks_loading.get_auth_plugin_conf_options('password') +
+            ks_loading.get_auth_plugin_conf_options('v2password') +
+            ks_loading.get_auth_plugin_conf_options('v3password'))
+    }

cyborg/context.py

@@ -0,0 +1,171 @@
+# Copyright 2011 OpenStack Foundation
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from keystoneauth1.access import service_catalog as ksa_service_catalog
+from keystoneauth1 import plugin
+from oslo_context import context
+from oslo_db.sqlalchemy import enginefacade
+from oslo_utils import timeutils
+import six
+
+
+class _ContextAuthPlugin(plugin.BaseAuthPlugin):
+    """A keystoneauth auth plugin that uses the values from the Context.
+
+    Ideally we would use the plugin provided by auth_token middleware however
+    this plugin isn't serialized yet so we construct one from the serialized
+    auth data.
+    """
+
+    def __init__(self, auth_token, sc):
+        super(_ContextAuthPlugin, self).__init__()
+
+        self.auth_token = auth_token
+        self.service_catalog = ksa_service_catalog.ServiceCatalogV2(sc)
+
+    def get_token(self, *args, **kwargs):
+        return self.auth_token
+
+    def get_endpoint(self, session, service_type=None, interface=None,
+                     region_name=None, service_name=None, **kwargs):
+        return self.service_catalog.url_for(service_type=service_type,
+                                            service_name=service_name,
+                                            interface=interface,
+                                            region_name=region_name)
+
+
+@enginefacade.transaction_context_provider
+class RequestContext(context.RequestContext):
+    """Security context and request information.
+
+    Represents the user taking a given action within the system.
+
+    """
+
+    def __init__(self, user_id=None, project_id=None, is_admin=None,
+                 read_deleted="no", remote_address=None, timestamp=None,
+                 quota_class=None, service_catalog=None,
+                 user_auth_plugin=None, **kwargs):
+        """:param read_deleted: 'no' indicates deleted records are hidden,
+                'yes' indicates deleted records are visible,
+                'only' indicates that *only* deleted records are visible.
+
+           :param overwrite: Set to False to ensure that the greenthread local
+                copy of the index is not overwritten.
+
+           :param instance_lock_checked: This is not used and will be removed
+                in a future release.
+
+           :param user_auth_plugin: The auth plugin for the current request's
+                authentication data.
+        """
+        if user_id:
+            kwargs['user_id'] = user_id
+        if project_id:
+            kwargs['project_id'] = project_id
+
+        super(RequestContext, self).__init__(is_admin=is_admin, **kwargs)
+
+        self.read_deleted = read_deleted
+        self.remote_address = remote_address
+        if not timestamp:
+            timestamp = timeutils.utcnow()
+        if isinstance(timestamp, six.string_types):
+            timestamp = timeutils.parse_strtime(timestamp)
+        self.timestamp = timestamp
+
+        if service_catalog:
+            # Only include required parts of service_catalog
+            self.service_catalog = [s for s in service_catalog
+                                    if s.get('type') in ('image')]
+        else:
+            # if list is empty or none
+            self.service_catalog = []
+
+        self.user_auth_plugin = user_auth_plugin
+        # if self.is_admin is None:
+        #    self.is_admin = policy.check_is_admin(self)
+
+    def get_auth_plugin(self):
+        if self.user_auth_plugin:
+            return self.user_auth_plugin
+        else:
+            return _ContextAuthPlugin(self.auth_token, self.service_catalog)
+
+
+def get_context():
+    """A helper method to get a blank context.
+
+    Note that overwrite is False here so this context will not update the
+    greenthread-local stored context that is used when logging.
+    """
+    return RequestContext(user_id=None,
+                          project_id=None,
+                          is_admin=False,
+                          overwrite=False)
+
+
+def get_admin_context(read_deleted="no"):
+    # NOTE(alaski): This method should only be used when an admin context is
+    # necessary for the entirety of the context lifetime. If that's not the
+    # case please use get_context(), or create the RequestContext manually, and
+    # use context.elevated() where necessary. Some periodic tasks may use
+    # get_admin_context so that their database calls are not filtered on
+    # project_id.
+    return RequestContext(user_id=None,
+                          project_id=None,
+                          is_admin=True,
+                          read_deleted=read_deleted,
+                          overwrite=False)
+
+
+def is_user_context(context):
+    """Indicates if the request context is a normal user."""
+    if not context:
+        return False
+    if context.is_admin:
+        return False
+    if not context.user_id or not context.project_id:
+        return False
+    return True
+
+
+def require_context(ctxt):
+    """Raise exception.Forbidden() if context is not a user or an
+    admin context.
+    """
+    if not ctxt.is_admin and not is_user_context(ctxt):
+        raise exception.Forbidden()
+
+
+def authorize_project_context(context, project_id):
+    """Ensures a request has permission to access the given project."""
+    if is_user_context(context):
+        if not context.project_id:
+            raise exception.Forbidden()
+        elif context.project_id != project_id:
+            raise exception.Forbidden()
+
+
+def authorize_user_context(context, user_id):
+    """Ensures a request has permission to access the given user."""
+    if is_user_context(context):
+        if not context.user_id:
+            raise exception.Forbidden()
+        elif context.user_id != user_id:
+            raise exception.Forbidden()

cyborg/image/api.py

@@ -0,0 +1,163 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Main abstraction layer for retrieving and storing information about acclerator
+images used by the cyborg agent layer.
+"""
+
+from cyborg.image import glance
+from oslo_log import log
+
+LOG = log.getLogger(__name__)
+
+
+class API(object):
+
+    """Responsible for exposing a relatively stable internal API for other
+    modules in Cyborg to retrieve information about acclerator images.
+    """
+
+    def _get_session_and_image_id(self, context, id_or_uri):
+        """Returns a tuple of (session, image_id). If the supplied `id_or_uri`
+        is an image ID, then the default client session will be returned
+        for the context's user, along with the image ID. If the supplied
+        `id_or_uri` parameter is a URI, then a client session connecting to
+        the URI's image service endpoint will be returned along with a
+        parsed image ID from that URI.
+
+        :param context: The `cyborg.context.Context` object for the request
+        :param id_or_uri: A UUID identifier or an image URI to look up image
+                          information for.
+        """
+        return glance.get_remote_image_service(context, id_or_uri)
+
+    def _get_session(self, _context):
+        """Returns a client session that can be used to query for image
+        information.
+
+        :param _context: The `cyborg.context.Context` object for the request
+        """
+        return glance.get_default_image_service()
+
+    @staticmethod
+    def generate_image_url(image_ref, context):
+        """Generate an image URL from an image_ref.
+
+        :param image_ref: The image ref to generate URL
+        :param context: The `cyborg.context.Context` object for the request
+        """
+        return "%s/images/%s" % (next(glance.get_api_servers(context)),
+                                 image_ref)
+
+    def get_all(self, context, **kwargs):
+        """Retrieves all information records about all acclerator images
+        available to show to the requesting user. If the requesting user is an
+        admin, all images in an ACTIVE status are returned. If the requesting
+        user is not an admin, the all public images and all private images
+        that are owned by the requesting user in the ACTIVE status are
+        returned.
+
+        :param context: The `cyborg.context.Context` object for the request
+        :param kwargs: A dictionary of filter and pagination values that
+                       may be passed to the underlying image info driver.
+        """
+        session = self._get_session(context)
+        return session.detail(context, **kwargs)
+
+    def get(self, context, id_or_uri):
+        """Retrieves the information record for a single acclerator image.
+        If the supplied identifier parameter is a UUID, the default driver will
+        be used to return information about the image. If the supplied
+        identifier is a URI, then the driver that matches that URI endpoint
+        will be used to query for image information.
+
+        :param context: The `cyborg.context.Context` object for the request
+        :param id_or_uri: A UUID identifier or an image URI to look up image
+                          information for.
+        """
+        session, image_id = self._get_session_and_image_id(context, id_or_uri)
+        return session.show(context, image_id,
+                            include_locations=False,
+                            show_deleted=False)
+
+    def update(self, context, id_or_uri, image_info,
+               data=None, purge_props=False):
+        """Update the information about an image, optionally along with a file
+        handle or bytestream iterator for image bits. If the optional file
+        handle for updated image bits is supplied, the image may not have
+        already uploaded bits for the image.
+
+        :param context: The `cyborg.context.Context` object for the request
+        :param id_or_uri: A UUID identifier or an image URI to look up image
+                          information for.
+        :param image_info: A dict of information about the image that is
+                           passed to the image registry.
+        :param data: Optional file handle or bytestream iterator that is
+                     passed to backend storage.
+        :param purge_props: Optional, defaults to False. If set, the backend
+                            image registry will clear all image properties
+                            and replace them the image properties supplied
+                            in the image_info dictionary's 'properties'
+                            collection.
+        """
+        session, image_id = self._get_session_and_image_id(context, id_or_uri)
+        return session.update(context, image_id, image_info, data=data,
+                              purge_props=purge_props)
+
+    def delete(self, context, id_or_uri):
+        """Delete the information about an image and mark the image bits for
+        deletion.
+
+        :param context: The `cyborg.context.Context` object for the request
+        :param id_or_uri: A UUID identifier or an image URI to look up image
+                          information for.
+        """
+        session, image_id = self._get_session_and_image_id(context, id_or_uri)
+        return session.delete(context, image_id)
+
+    def download(self, context, id_or_uri, data=None, dest_path=None):
+        """Transfer image bits from Glance or a known source location to the
+        supplied destination filepath.
+
+        :param context: The `cyborg.context.RequestContext` object for the
+                        request
+        :param id_or_uri: A UUID identifier or an image URI to look up image
+                          information for.
+        :param data: A file object to use in downloading image data.
+        :param dest_path: Filepath to transfer image bits to.
+
+        Note that because of the poor design of the
+        `glance.ImageService.download` method, the function returns different
+        things depending on what arguments are passed to it. If a data argument
+        is supplied but no dest_path is specified (only done in the XenAPI virt
+        driver's image.utils module) then None is returned from the method. If
+        the data argument is not specified but a destination path *is*
+        specified, then a writeable file handle to the destination path is
+        constructed in the method and the image bits written to that file, and
+        again, None is returned from the method. If no data argument is
+        supplied and no dest_path argument is supplied (VMWare and XenAPI virt
+        drivers), then the method returns an iterator to the image bits that
+        the caller uses to write to wherever location it wants. Finally, if the
+        allow_direct_url_schemes CONF option is set to something, then the
+        cyborg.image.download modules are used to attempt to do an SCP copy of
+        the image bits from a file location to the dest_path and None is
+        returned after retrying one or more download locations.
+
+        I think the above points to just how hacky/wacky all of this code is,
+        and the reason it needs to be cleaned up and standardized across the
+        virt driver callers.
+        """
+
+        session, image_id = self._get_session_and_image_id(context, id_or_uri)
+        return session.download(context, image_id, data=data,
+                                dst_path=dest_path)

cyborg/image/download/__init__.py

@@ -0,0 +1,54 @@
+# Copyright 2013 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log as logging
+import stevedore.driver
+import stevedore.extension
+
+LOG = logging.getLogger(__name__)
+
+
+def load_transfer_modules():
+
+    module_dictionary = {}
+
+    ex = stevedore.extension.ExtensionManager('cyborg.image.download.modules')
+    for module_name in ex.names():
+        mgr = stevedore.driver.DriverManager(
+            namespace='cyborg.image.download.modules',
+            name=module_name,
+            invoke_on_load=False)
+
+        schemes_list = mgr.driver.get_schemes()
+        for scheme in schemes_list:
+            if scheme in module_dictionary:
+                LOG.error('%(scheme)s is registered as a module twice. '
+                          '%(module_name)s is not being used.',
+                          {'scheme': scheme,
+                           'module_name': module_name})
+            else:
+                module_dictionary[scheme] = mgr.driver
+
+    if module_dictionary:
+        LOG.warning('The cyborg.image.download.modules extension point is '
+                    'deprecated for removal starting in the 17.0.0 Queens '
+                    'release and may be removed as early as the 18.0.0 Rocky '
+                    'release. It is not maintained and there is no indication '
+                    'of its use in production clouds. If you are using this '
+                    'extension point, please make the cyborg development team '
+                    'aware by contacting us in the #openstack-cyborg freenode '
+                    'IRC channel or on the openstack-dev mailing list.')
+
+    return module_dictionary

cyborg/image/glance.py

@@ -0,0 +1,707 @@
+# Copyright 2010 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Implementation of an image service that uses Glance as the backend."""
+
+from __future__ import absolute_import
+
+import copy
+import inspect
+import itertools
+import os
+import random
+import re
+import stat
+import sys
+import time
+
+import cryptography
+from cursive import exception as cursive_exception
+from cursive import signature_utils
+import glanceclient
+import glanceclient.exc
+from glanceclient.v2 import schemas
+from keystoneauth1 import loading as ks_loading
+from oslo_log import log as logging
+from oslo_serialization import jsonutils
+from oslo_utils import excutils
+from oslo_utils import timeutils
+import six
+from six.moves import range
+import six.moves.urllib.parse as urlparse
+
+import cyborg.conf
+from cyborg.common import exception
+import cyborg.image.download as image_xfers
+from cyborg import objects
+from cyborg.objects import fields
+from cyborg import service_auth
+from cyborg.common import utils
+
+
+LOG = logging.getLogger(__name__)
+CONF = cyborg.conf.CONF
+
+_SESSION = None
+
+
+def _session_and_auth(context):
+    # Session is cached, but auth needs to be pulled from context each time.
+    global _SESSION
+
+    if not _SESSION:
+        _SESSION = ks_loading.load_session_from_conf_options(
+            CONF, cyborg.conf.glance.glance_group.name)
+
+    auth = service_auth.get_auth_plugin(context)
+
+    return _SESSION, auth
+
+
+def _glanceclient_from_endpoint(context, endpoint, version):
+    sess, auth = _session_and_auth(context)
+
+    return glanceclient.Client(version, session=sess, auth=auth,
+                               endpoint_override=endpoint,
+                               global_request_id=context.global_id)
+
+
+def generate_glance_url(context):
+    """Return a random glance url from the api servers we know about."""
+    return next(get_api_servers(context))
+
+
+def _endpoint_from_image_ref(image_href):
+    """Return the image_ref and guessed endpoint from an image url.
+
+    :param image_href: href of an image
+    :returns: a tuple of the form (image_id, endpoint_url)
+    """
+    parts = image_href.split('/')
+    image_id = parts[-1]
+    # the endpoint is everything in the url except the last 3 bits
+    # which are version, 'images', and image_id
+    endpoint = '/'.join(parts[:-3])
+    return (image_id, endpoint)
+
+
+def generate_identity_headers(context, status='Confirmed'):
+    return {
+        'X-Auth-Token': getattr(context, 'auth_token', None),
+        'X-User-Id': getattr(context, 'user_id', None),
+        'X-Tenant-Id': getattr(context, 'project_id', None),
+        'X-Roles': ','.join(getattr(context, 'roles', [])),
+        'X-Identity-Status': status,
+    }
+
+
+def get_api_servers(context):
+    """Shuffle a list of service endpoints and return an iterator that will
+    cycle through the list, looping around to the beginning if necessary.
+    """
+    # NOTE(efried): utils.get_ksa_adapter().get_endpoint() is the preferred
+    # mechanism for endpoint discovery. Only use `api_servers` if you really
+    # need to shuffle multiple endpoints.
+    if CONF.glance.api_servers:
+        api_servers = CONF.glance.api_servers
+        random.shuffle(api_servers)
+    else:
+        sess, auth = _session_and_auth(context)
+        ksa_adap = utils.get_ksa_adapter(
+            cyborg.conf.glance.DEFAULT_SERVICE_TYPE,
+            ksa_auth=auth, ksa_session=sess,
+            min_version='2.0', max_version='2.latest')
+        endpoint = utils.get_endpoint(ksa_adap)
+        if endpoint:
+            # NOTE(mriedem): Due to python-glanceclient bug 1707995 we have
+            # to massage the endpoint URL otherwise it won't work properly.
+            # We can't use glanceclient.common.utils.strip_version because
+            # of bug 1748009.
+            endpoint = re.sub(r'/v\d+(\.\d+)?/?$', '/', endpoint)
+        api_servers = [endpoint]
+
+    return itertools.cycle(api_servers)
+
+
+class GlanceClientWrapper(object):
+    """Glance client wrapper class that implements retries."""
+
+    def __init__(self, context=None, endpoint=None):
+        version = 2
+        if endpoint is not None:
+            self.client = self._create_static_client(context,
+                                                     endpoint,
+                                                     version)
+        else:
+            self.client = None
+        self.api_servers = None
+
+    def _create_static_client(self, context, endpoint, version):
+        """Create a client that we'll use for every call."""
+        self.api_server = str(endpoint)
+        return _glanceclient_from_endpoint(context, endpoint, version)
+
+    def _create_onetime_client(self, context, version):
+        """Create a client that will be used for one call."""
+        if self.api_servers is None:
+            self.api_servers = get_api_servers(context)
+        self.api_server = next(self.api_servers)
+        return _glanceclient_from_endpoint(context, self.api_server, version)
+
+    def call(self, context, version, method, *args, **kwargs):
+        """Call a glance client method.  If we get a connection error,
+        retry the request according to CONF.glance.num_retries.
+        """
+        retry_excs = (glanceclient.exc.ServiceUnavailable,
+                      glanceclient.exc.InvalidEndpoint,
+                      glanceclient.exc.CommunicationError)
+        num_attempts = 1 + CONF.glance.num_retries
+
+        for attempt in range(1, num_attempts + 1):
+            client = self.client or self._create_onetime_client(context,
+                                                                version)
+            try:
+                controller = getattr(client,
+                                     kwargs.pop('controller', 'images'))
+                result = getattr(controller, method)(*args, **kwargs)
+                if inspect.isgenerator(result):
+                    # Convert generator results to a list, so that we can
+                    # catch any potential exceptions now and retry the call.
+                    return list(result)
+                return result
+            except retry_excs as e:
+                if attempt < num_attempts:
+                    extra = "retrying"
+                else:
+                    extra = 'done trying'
+
+                LOG.exception("Error contacting glance server "
+                              "'%(server)s' for '%(method)s', "
+                              "%(extra)s.",
+                              {'server': self.api_server,
+                               'method': method, 'extra': extra})
+                if attempt == num_attempts:
+                    raise exception.GlanceConnectionFailed(
+                        server=str(self.api_server), reason=six.text_type(e))
+                time.sleep(1)
+
+
+class GlanceImageServiceV2(object):
+    """Provides storage and retrieval of disk image objects within Glance."""
+
+    def __init__(self, client=None):
+        self._client = client or GlanceClientWrapper()
+        # NOTE(jbresnah) build the table of download handlers at the beginning
+        # so that operators can catch errors at load time rather than whenever
+        # a user attempts to use a module.  Note this cannot be done in glance
+        # space when this python module is loaded because the download module
+        # may require configuration options to be parsed.
+        self._download_handlers = {}
+        download_modules = image_xfers.load_transfer_modules()
+
+        for scheme, mod in download_modules.items():
+            if scheme not in CONF.glance.allowed_direct_url_schemes:
+                continue
+
+            try:
+                self._download_handlers[scheme] = mod.get_download_handler()
+            except Exception as ex:
+                LOG.error('When loading the module %(module_str)s the '
+                          'following error occurred: %(ex)s',
+                          {'module_str': str(mod), 'ex': ex})
+
+    @staticmethod
+    def _safe_fsync(fh):
+        """Performs os.fsync on a filehandle only if it is supported.
+
+        fsync on a pipe, FIFO, or socket raises OSError with EINVAL.  This
+        method discovers whether the target filehandle is one of these types
+        and only performs fsync if it isn't.
+
+        :param fh: Open filehandle (not a path or fileno) to maybe fsync.
+        """
+        fileno = fh.fileno()
+        mode = os.fstat(fileno).st_mode
+        # A pipe answers True to S_ISFIFO
+        if not any(check(mode) for check in (stat.S_ISFIFO, stat.S_ISSOCK)):
+            os.fsync(fileno)
+
+    def download(self, context, image_id, data=None, dst_path=None):
+        """Calls out to Glance for data and writes data."""
+        if CONF.glance.allowed_direct_url_schemes and dst_path is not None:
+            image = self.show(context, image_id, include_locations=True)
+            for entry in image.get('locations', []):
+                loc_url = entry['url']
+                loc_meta = entry['metadata']
+                o = urlparse.urlparse(loc_url)
+                xfer_mod = self._get_transfer_module(o.scheme)
+                if xfer_mod:
+                    try:
+                        xfer_mod.download(context, o, dst_path, loc_meta)
+                        LOG.info("Successfully transferred using %s", o.scheme)
+                        return
+                    except Exception:
+                        LOG.exception("Download image error")
+
+        try:
+            image_chunks = self._client.call(context, 2, 'data', image_id)
+        except Exception:
+            _reraise_translated_image_exception(image_id)
+
+        if image_chunks.wrapped is None:
+            # None is a valid return value, but there's nothing we can do with
+            # a image with no associated data
+            raise exception.ImageUnacceptable(image_id=image_id,
+                                              reason='Image has no \
+                                              associated data')
+
+        # Retrieve properties for verification of Glance image signature
+        verifier = None
+        if CONF.glance.verify_glance_signatures:
+            image_meta_dict = self.show(context, image_id,
+                                        include_locations=False)
+            image_meta = objects.ImageMeta.from_dict(image_meta_dict)
+            img_signature = image_meta.properties.get('img_signature')
+            img_sig_hash_method = image_meta.properties.get(
+                'img_signature_hash_method'
+            )
+            img_sig_cert_uuid = image_meta.properties.get(
+                'img_signature_certificate_uuid'
+            )
+            img_sig_key_type = image_meta.properties.get(
+                'img_signature_key_type'
+            )
+            try:
+                verifier = signature_utils.get_verifier(
+                    context=context,
+                    img_signature_certificate_uuid=img_sig_cert_uuid,
+                    img_signature_hash_method=img_sig_hash_method,
+                    img_signature=img_signature,
+                    img_signature_key_type=img_sig_key_type,
+                )
+            except cursive_exception.SignatureVerificationError:
+                with excutils.save_and_reraise_exception():
+                    LOG.error('Image signature verification failed '
+                              'for image: %s', image_id)
+
+        close_file = False
+        if data is None and dst_path:
+            data = open(dst_path, 'wb')
+            close_file = True
+
+        if data is None:
+
+            # Perform image signature verification
+            if verifier:
+                try:
+                    for chunk in image_chunks:
+                        verifier.update(chunk)
+                    verifier.verify()
+
+                    LOG.info('Image signature verification succeeded '
+                             'for image: %s', image_id)
+
+                except cryptography.exceptions.InvalidSignature:
+                    with excutils.save_and_reraise_exception():
+                        LOG.error('Image signature verification failed '
+                                  'for image: %s', image_id)
+            return image_chunks
+        else:
+            try:
+                for chunk in image_chunks:
+                    if verifier:
+                        verifier.update(chunk)
+                    data.write(chunk)
+                if verifier:
+                    verifier.verify()
+                    LOG.info('Image signature verification succeeded '
+                             'for image %s', image_id)
+            except cryptography.exceptions.InvalidSignature:
+                data.truncate(0)
+                with excutils.save_and_reraise_exception():
+                    LOG.error('Image signature verification failed '
+                              'for image: %s', image_id)
+            except Exception as ex:
+                with excutils.save_and_reraise_exception():
+                    LOG.error("Error writing to %(path)s: %(exception)s",
+                              {'path': dst_path, 'exception': ex})
+            finally:
+                if close_file:
+                    # Ensure that the data is pushed all the way down to
+                    # persistent storage. This ensures that in the event of a
+                    # subsequent host crash we don't have running instances
+                    # using a corrupt backing file.
+                    data.flush()
+                    self._safe_fsync(data)
+                    data.close()
+
+
+def _extract_query_params(params):
+    _params = {}
+    accepted_params = ('filters', 'marker', 'limit',
+                       'page_size', 'sort_key', 'sort_dir')
+    for param in accepted_params:
+        if params.get(param):
+            _params[param] = params.get(param)
+
+    # ensure filters is a dict
+    _params.setdefault('filters', {})
+    # NOTE(vish): don't filter out private images
+    _params['filters'].setdefault('is_public', 'none')
+
+    return _params
+
+
+def _extract_query_params_v2(params):
+    _params = {}
+    accepted_params = ('filters', 'marker', 'limit',
+                       'page_size', 'sort_key', 'sort_dir')
+    for param in accepted_params:
+        if params.get(param):
+            _params[param] = params.get(param)
+
+    # ensure filters is a dict
+    _params.setdefault('filters', {})
+    # NOTE(vish): don't filter out private images
+    _params['filters'].setdefault('is_public', 'none')
+
+    # adopt filters to be accepted by glance v2 api
+    filters = _params['filters']
+    new_filters = {}
+
+    for filter_ in filters:
+        # remove 'property-' prefix from filters by custom properties
+        if filter_.startswith('property-'):
+            new_filters[filter_.lstrip('property-')] = filters[filter_]
+        elif filter_ == 'changes-since':
+            # convert old 'changes-since' into new 'updated_at' filter
+            updated_at = 'gte:' + filters['changes-since']
+            new_filters['updated_at'] = updated_at
+        elif filter_ == 'is_public':
+            # convert old 'is_public' flag into 'visibility' filter
+            # omit the filter if is_public is None
+            is_public = filters['is_public']
+            if is_public.lower() in ('true', '1'):
+                new_filters['visibility'] = 'public'
+            elif is_public.lower() in ('false', '0'):
+                new_filters['visibility'] = 'private'
+        else:
+            new_filters[filter_] = filters[filter_]
+
+    _params['filters'] = new_filters
+
+    return _params
+
+
+def _is_image_available(context, image):
+    """Check image availability.
+
+    This check is needed in case cyborg and Glance are deployed
+    without authentication turned on.
+    """
+    # The presence of an auth token implies this is an authenticated
+    # request and we need not handle the noauth use-case.
+    if hasattr(context, 'auth_token') and context.auth_token:
+        return True
+
+    def _is_image_public(image):
+        # NOTE(jaypipes) V2 Glance API replaced the is_public attribute
+        # with a visibility attribute. We do this here to prevent the
+        # glanceclient for a V2 image model from throwing an
+        # exception from warlock when trying to access an is_public
+        # attribute.
+        if hasattr(image, 'visibility'):
+            return str(image.visibility).lower() == 'public'
+        else:
+            return image.is_public
+
+    if context.is_admin or _is_image_public(image):
+        return True
+
+    properties = image.properties
+
+    if context.project_id and ('owner_id' in properties):
+        return str(properties['owner_id']) == str(context.project_id)
+
+    if context.project_id and ('project_id' in properties):
+        return str(properties['project_id']) == str(context.project_id)
+
+    try:
+        user_id = properties['user_id']
+    except KeyError:
+        return False
+
+    return str(user_id) == str(context.user_id)
+
+
+def _translate_to_glance(image_meta):
+    image_meta = _convert_to_string(image_meta)
+    image_meta = _remove_read_only(image_meta)
+    image_meta = _convert_to_v2(image_meta)
+    return image_meta
+
+
+def _convert_to_v2(image_meta):
+    output = {}
+    for name, value in image_meta.items():
+        if name == 'properties':
+            for prop_name, prop_value in value.items():
+                # if allow_additional_image_properties is disabled we can't
+                # define kernel_id and ramdisk_id as None, so we have to omit
+                # these properties if they are not set.
+                if prop_name in ('kernel_id', 'ramdisk_id') and \
+                   prop_value is not None and \
+                   prop_value.strip().lower() in ('none', ''):
+                    continue
+                # in glance only string and None property values are allowed,
+                # v1 client accepts any values and converts them to string,
+                # v2 doesn't - so we have to take care of it.
+                elif prop_value is None or isinstance(
+                        prop_value, six.string_types):
+                    output[prop_name] = prop_value
+                else:
+                    output[prop_name] = str(prop_value)
+
+        elif name in ('min_ram', 'min_disk'):
+            output[name] = int(value)
+        elif name == 'is_public':
+            output['visibility'] = 'public' if value else 'private'
+        elif name in ('size', 'deleted'):
+            continue
+        else:
+            output[name] = value
+    return output
+
+
+def _translate_from_glance(image, include_locations=False):
+    image_meta = _extract_attributes_v2(
+        image, include_locations=include_locations)
+
+    image_meta = _convert_timestamps_to_datetimes(image_meta)
+    image_meta = _convert_from_string(image_meta)
+    return image_meta
+
+
+def _convert_timestamps_to_datetimes(image_meta):
+    """Returns image with timestamp fields converted to datetime objects."""
+    for attr in ['created_at', 'updated_at', 'deleted_at']:
+        if image_meta.get(attr):
+            image_meta[attr] = timeutils.parse_isotime(image_meta[attr])
+    return image_meta
+
+
+# NOTE(bcwaldon): used to store non-string data in glance metadata
+def _json_loads(properties, attr):
+    prop = properties[attr]
+    if isinstance(prop, six.string_types):
+        properties[attr] = jsonutils.loads(prop)
+
+
+def _json_dumps(properties, attr):
+    prop = properties[attr]
+    if not isinstance(prop, six.string_types):
+        properties[attr] = jsonutils.dumps(prop)
+
+
+_CONVERT_PROPS = ('block_device_mapping', 'mappings')
+
+
+def _convert(method, metadata):
+    metadata = copy.deepcopy(metadata)
+    properties = metadata.get('properties')
+    if properties:
+        for attr in _CONVERT_PROPS:
+            if attr in properties:
+                method(properties, attr)
+
+    return metadata
+
+
+def _convert_from_string(metadata):
+    return _convert(_json_loads, metadata)
+
+
+def _convert_to_string(metadata):
+    return _convert(_json_dumps, metadata)
+
+
+def _extract_attributes(image, include_locations=False):
+    # TODO(mfedosin): Remove this function once we move to glance V2
+    # completely.
+    # NOTE(hdd): If a key is not found, base.Resource.__getattr__() may perform
+    # a get(), resulting in a useless request back to glance. This list is
+    # therefore sorted, with dependent attributes as the end
+    # 'deleted_at' depends on 'deleted'
+    # 'checksum' depends on 'status' == 'active'
+    IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner',
+                        'container_format', 'status', 'id',
+                        'name', 'created_at', 'updated_at',
+                        'deleted', 'deleted_at', 'checksum',
+                        'min_disk', 'min_ram', 'is_public',
+                        'direct_url', 'locations']
+
+    queued = getattr(image, 'status') == 'queued'
+    queued_exclude_attrs = ['disk_format', 'container_format']
+    include_locations_attrs = ['direct_url', 'locations']
+    output = {}
+
+    for attr in IMAGE_ATTRIBUTES:
+        if attr == 'deleted_at' and not output['deleted']:
+            output[attr] = None
+        elif attr == 'checksum' and output['status'] != 'active':
+            output[attr] = None
+        # image may not have 'name' attr
+        elif attr == 'name':
+            output[attr] = getattr(image, attr, None)
+        # NOTE(liusheng): queued image may not have these attributes and 'name'
+        elif queued and attr in queued_exclude_attrs:
+            output[attr] = getattr(image, attr, None)
+        # NOTE(mriedem): Only get location attrs if including locations.
+        elif attr in include_locations_attrs:
+            if include_locations:
+                output[attr] = getattr(image, attr, None)
+        # NOTE(mdorman): 'size' attribute must not be 'None', so use 0 instead
+        elif attr == 'size':
+            # NOTE(mriedem): A snapshot image may not have the size attribute
+            # set so default to 0.
+            output[attr] = getattr(image, attr, 0) or 0
+        else:
+            # NOTE(xarses): Anything that is caught with the default value
+            # will result in an additional lookup to glance for said attr.
+            # Notable attributes that could have this issue:
+            # disk_format, container_format, name, deleted, checksum
+            output[attr] = getattr(image, attr, None)
+
+    output['properties'] = getattr(image, 'properties', {})
+
+    return output
+
+
+def _extract_attributes_v2(image, include_locations=False):
+    include_locations_attrs = ['direct_url', 'locations']
+    omit_attrs = ['self', 'schema', 'protected', 'virtual_size', 'file',
+                  'tags']
+    raw_schema = image.schema
+    schema = schemas.Schema(raw_schema)
+    output = {'properties': {}, 'deleted': False, 'deleted_at': None,
+              'disk_format': None, 'container_format': None, 'name': None,
+              'checksum': None}
+    for name, value in image.items():
+        if (name in omit_attrs
+                or name in include_locations_attrs and not include_locations):
+            continue
+        elif name == 'visibility':
+            output['is_public'] = value == 'public'
+        elif name == 'size' and value is None:
+            output['size'] = 0
+        elif schema.is_base_property(name):
+            output[name] = value
+        else:
+            output['properties'][name] = value
+
+    return output
+
+
+def _remove_read_only(image_meta):
+    IMAGE_ATTRIBUTES = ['status', 'updated_at', 'created_at', 'deleted_at']
+    output = copy.deepcopy(image_meta)
+    for attr in IMAGE_ATTRIBUTES:
+        if attr in output:
+            del output[attr]
+    return output
+
+
+def _reraise_translated_image_exception(image_id):
+    """Transform the exception for the image but keep its traceback intact."""
+    exc_type, exc_value, exc_trace = sys.exc_info()
+    new_exc = _translate_image_exception(image_id, exc_value)
+    six.reraise(type(new_exc), new_exc, exc_trace)
+
+
+def _reraise_translated_exception():
+    """Transform the exception but keep its traceback intact."""
+    exc_type, exc_value, exc_trace = sys.exc_info()
+    new_exc = _translate_plain_exception(exc_value)
+    six.reraise(type(new_exc), new_exc, exc_trace)
+
+
+def _translate_image_exception(image_id, exc_value):
+    if isinstance(exc_value, (glanceclient.exc.Forbidden,
+                  glanceclient.exc.Unauthorized)):
+        return exception.ImageNotAuthorized(image_id=image_id)
+    if isinstance(exc_value, glanceclient.exc.NotFound):
+        return exception.ImageNotFound(image_id=image_id)
+    if isinstance(exc_value, glanceclient.exc.BadRequest):
+        return exception.ImageBadRequest(image_id=image_id,
+                                         response=six.text_type(exc_value))
+    return exc_value
+
+
+def _translate_plain_exception(exc_value):
+    if isinstance(exc_value, (glanceclient.exc.Forbidden,
+                  glanceclient.exc.Unauthorized)):
+        return exception.Forbidden(six.text_type(exc_value))
+    if isinstance(exc_value, glanceclient.exc.NotFound):
+        return exception.NotFound(six.text_type(exc_value))
+    if isinstance(exc_value, glanceclient.exc.BadRequest):
+        return exception.Invalid(six.text_type(exc_value))
+    return exc_value
+
+
+def get_remote_image_service(context, image_href):
+    """Create an image_service and parse the id from the given image_href.
+
+    The image_href param can be an href of the form
+    'http://example.com:9292/v1/images/b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3',
+    or just an id such as 'b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3'. If the
+    image_href is a standalone id, then the default image service is returned.
+
+    :param image_href: href that describes the location of an image
+    :returns: a tuple of the form (image_service, image_id)
+
+    """
+    # NOTE(bcwaldon): If image_href doesn't look like a URI, assume its a
+    # standalone image ID
+    if '/' not in str(image_href):
+        image_service = get_default_image_service()
+        return image_service, image_href
+
+    try:
+        (image_id, endpoint) = _endpoint_from_image_ref(image_href)
+        glance_client = GlanceClientWrapper(context=context,
+                                            endpoint=endpoint)
+    except ValueError:
+        raise exception.InvalidImageRef(image_href=image_href)
+
+    image_service = GlanceImageServiceV2(client=glance_client)
+    return image_service, image_id
+
+
+def get_default_image_service():
+    return GlanceImageServiceV2()
+
+
+class UpdateGlanceImage(object):
+    def __init__(self, context, image_id, metadata, stream):
+        self.context = context
+        self.image_id = image_id
+        self.metadata = metadata
+        self.image_stream = stream
+
+    def start(self):
+        image_service, image_id = (
+            get_remote_image_service(self.context, self.image_id))
+        image_service.update(self.context, image_id, self.metadata,
+                             self.image_stream, purge_props=False)

cyborg/service_auth.py

@@ -0,0 +1,54 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from keystoneauth1 import loading as ks_loading
+from keystoneauth1 import service_token
+from oslo_log import log as logging
+
+import cyborg.conf
+
+
+CONF = cyborg.conf.CONF
+LOG = logging.getLogger(__name__)
+
+_SERVICE_AUTH = None
+
+
+def reset_globals():
+    """For async unit test consistency."""
+    global _SERVICE_AUTH
+    _SERVICE_AUTH = None
+
+
+def get_auth_plugin(context):
+    user_auth = context.get_auth_plugin()
+
+    if CONF.service_user.send_service_user_token:
+        global _SERVICE_AUTH
+        if not _SERVICE_AUTH:
+            _SERVICE_AUTH = ks_loading.\
+                load_auth_from_conf_options(CONF,
+                                            group=cyborg.
+                                            conf.service_token.
+                                            SERVICE_USER_GROUP)
+            if _SERVICE_AUTH is None:
+                # This indicates a misconfiguration so log a warning and
+                # return the user_auth.
+                LOG.warning('Unable to load auth from [service_user] '
+                            'configuration. Ensure "auth_type" is set.')
+                return user_auth
+        return service_token.\
+            ServiceTokenAuthWrapper(user_auth=user_auth,
+                                    service_auth=_SERVICE_AUTH)
+
+    return user_auth