From c9e03967524486e2909ee18108f8004f1ff1774d Mon Sep 17 00:00:00 2001 From: Sanket Date: Tue, 22 Aug 2017 15:02:23 +0530 Subject: [PATCH] Add Azure support for Glance This drivers defines Glance location format for Azure images and support for adding info of Azure images inside glance. Change-Id: I68954be5b926b7f390b275c459484051618d8ebd Implements: blueprint azure-support --- glance/azure/create-glance-images-azure.py | 165 ++++++++++++++++++ .../glance_store/_drivers/azure/__init__.py | 0 glance/glance_store/_drivers/azure/config.py | 25 +++ glance/glance_store/_drivers/azure/store.py | 159 +++++++++++++++++ glance/glance_store/_drivers/azure/utils.py | 49 ++++++ 5 files changed, 398 insertions(+) create mode 100644 glance/azure/create-glance-images-azure.py create mode 100644 glance/glance_store/_drivers/azure/__init__.py create mode 100644 glance/glance_store/_drivers/azure/config.py create mode 100644 glance/glance_store/_drivers/azure/store.py create mode 100644 glance/glance_store/_drivers/azure/utils.py diff --git a/glance/azure/create-glance-images-azure.py b/glance/azure/create-glance-images-azure.py new file mode 100644 index 0000000..4171e59 --- /dev/null +++ b/glance/azure/create-glance-images-azure.py @@ -0,0 +1,165 @@ +""" +Copyright (c) 2017 Platform9 Systems Inc. (http://www.platform9.com) +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.compute import ComputeManagementClient +from functools import partial +from glanceclient import Client +from keystoneauth1 import loading +from keystoneauth1 import session + +import hashlib +import os +import sys +import uuid + + +def get_credentials(tenant_id, client_id, client_secret): + credentials = ServicePrincipalCredentials( + client_id=client_id, secret=client_secret, tenant=tenant_id) + return credentials + + +def _get_client(tenant_id, client_id, client_secret, subscription_id, + cls=None): + """Returns Azure compute resource object for interacting with Azure API + :param tenant_id: string, tenant_id from azure account + :param client_id: string, client_id (application id) + :param client_secret: string, secret key of application + :param subscription_id: string, unique identification id of account + :return: :class:`Resource ` object + """ + credentials = get_credentials(tenant_id, client_id, client_secret) + client = cls(credentials, subscription_id) + return client + + +get_compute_client = partial(_get_client, cls=ComputeManagementClient) + + +def abort(message): + sys.exit(message) + + +def get_env_param(env_name): + if env_name in os.environ: + return os.environ[env_name] + abort("%s environment variable not set." % env_name) + + +def get_keystone_session(vendor_data): + username = vendor_data['username'] + password = vendor_data['password'] + project_name = vendor_data['tenant_name'] + auth_url = vendor_data['auth_url'] + + loader = loading.get_plugin_loader('password') + auth = loader.load_from_options( + auth_url=auth_url, project_name=project_name, + username=username, password=password) + sess = session.Session(auth=auth) + return sess + + +def get_glance_client(vendor_data): + GLANCE_VERSION = '2' + glance_client = Client(GLANCE_VERSION, + session=get_keystone_session(vendor_data)) + return glance_client + + +class GlanceOperator(object): + def __init__(self): + auth_url = get_env_param('OS_AUTH_URL') + project_name = os.environ.get('OS_PROJECT_NAME') + tenant_name = os.environ.get('OS_TENANT_NAME') + username = get_env_param('OS_USERNAME') + password = get_env_param('OS_PASSWORD') + if not project_name: + if not tenant_name: + raise Exception("Either OS_PROJECT_NAME or OS_TENANT_NAME is " + "required.") + project_name = tenant_name + self.vendor_data = {'username': username, + 'password': password, + 'auth_url': auth_url, + 'tenant_name': project_name} + self.glance_client = get_glance_client(self.vendor_data) + + def register_image(self, image): + locations = image.pop('locations') + response = self.glance_client.images.create(**image) + glance_id = response['id'] + for location in locations: + self.glance_client.images.add_location(glance_id, location['url'], + location['metadata']) + print("Registered image %s" % image['name']) + + +class ImageProvider(object): + def __init__(self): + self.glance_operator = GlanceOperator() + + def get_public_images(self): + raise NotImplementedError() + + def register_images(self): + for image_info in self.get_public_images(): + self.glance_operator.register_image(image_info) + + +class AzureImages(ImageProvider): + def __init__(self): + super(AzureImages, self).__init__() + tenant_id = get_env_param('AZURE_TENANT_ID') + client_id = get_env_param('AZURE_CLIENT_ID') + client_secret = get_env_param('AZURE_CLIENT_SECRET') + subscription_id = get_env_param('AZURE_SUBSCRIPTION_ID') + self.region = get_env_param('AZURE_REGION') + self.resource_group = get_env_param('AZURE_RESOURCE_GROUP') + self.compute_client = get_compute_client( + tenant_id, client_id, client_secret, subscription_id) + + def _azure_to_openstack_formatter(self, image_info): + """Converts Azure image data to Openstack image data format.""" + image_uuid = self._get_image_uuid(image_info.id) + location_info = [ + { + 'url': 'azure://{0}/{1}'.format(image_info.id.strip('/'), + image_uuid), + 'metadata': {'azure_link': image_info.id} + }, + ] + return {'id': image_uuid, + 'name': image_info.name, + 'container_format': 'bare', + 'disk_format': 'raw', + 'visibility': 'public', + 'azure_link': image_info.id, + 'locations': location_info} + + def _get_image_uuid(self, azure_id): + md = hashlib.md5() + md.update(azure_id) + return str(uuid.UUID(bytes=md.digest())) + + def get_public_images(self): + images = self.compute_client.images + response = images.list_by_resource_group(self.resource_group) + for result in response.advance_page(): + image_response = images.get(self.resource_group, result.name) + yield self._azure_to_openstack_formatter(image_response) + + +if __name__ == '__main__': + az_images = AzureImages() + az_images.register_images() diff --git a/glance/glance_store/_drivers/azure/__init__.py b/glance/glance_store/_drivers/azure/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/glance/glance_store/_drivers/azure/config.py b/glance/glance_store/_drivers/azure/config.py new file mode 100644 index 0000000..881e41d --- /dev/null +++ b/glance/glance_store/_drivers/azure/config.py @@ -0,0 +1,25 @@ +""" +Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com) +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" +from oslo_config import cfg + +azure_group = cfg.OptGroup(name='azure', + title='Options to connect to Azure cloud') + +azure_opts = [ + cfg.StrOpt('tenant_id', help='Tenant id of Azure account'), + cfg.StrOpt('client_id', help='Azure client id'), + cfg.StrOpt('client_secret', help='Azure Client secret', secret=True), + cfg.StrOpt('subscription_id', help='Azure subscription id'), + cfg.StrOpt('region', help='Azure region'), + cfg.StrOpt('resource_group', help="Azure resource group"), +] diff --git a/glance/glance_store/_drivers/azure/store.py b/glance/glance_store/_drivers/azure/store.py new file mode 100644 index 0000000..2a01df8 --- /dev/null +++ b/glance/glance_store/_drivers/azure/store.py @@ -0,0 +1,159 @@ +""" +Copyright (c) 2017 Platform9 Systems Inc. (http://www.platform9.com) +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" +import logging + +from oslo_utils import units + +from glance_store._drivers.azure import config +from glance_store._drivers.azure import utils +from glance_store import capabilities +from glance_store import driver +from glance_store import exceptions +from glance_store.i18n import _ +from glance_store import location +from six.moves import urllib + +LOG = logging.getLogger(__name__) + +MAX_REDIRECTS = 5 +STORE_SCHEME = 'azure' + + +class StoreLocation(location.StoreLocation): + """Class describing Azure URI.""" + uri_attrs = ['subscriptions', 'providers', 'resourcegroups', 'images'] + + def __init__(self, store_specs, conf): + super(StoreLocation, self).__init__(store_specs, conf) + self._sorted_uri_attrs = sorted(self.uri_attrs) + + def process_specs(self): + self.scheme = self.specs.get('scheme', STORE_SCHEME) + for attr in self.uri_attrs: + setattr(self, attr, self.specs.get(attr)) + + def get_uri(self): + _uri_path = [] + for attr in self.uri_attrs: + _uri_path.extend([attr.capitalize(), getattr(self, attr)]) + return "{0}://{1}/{2}".format(self.scheme, "/".join(_uri_path), + self.glance_id) + + def _parse_attrs(self, attrs_info): + attrs_list = attrs_info.strip('/').split('/') + self.glance_id = attrs_list.pop() + attrs_dict = { + attrs_list[i].lower(): attrs_list[i + 1] + for i in range(0, len(attrs_list), 2) + } + if self._sorted_uri_attrs != sorted(attrs_dict.keys()): + raise exceptions.BadStoreUri( + message="Image URI should contain required attributes") + for k, v in attrs_dict.items(): + setattr(self, k, v) + + def parse_uri(self, uri): + """Parse URLs based on Azure scheme """ + LOG.debug('Parse uri %s' % (uri, )) + if not uri.startswith('%s://' % STORE_SCHEME): + reason = (_("URI %(uri)s must start with %(scheme)s://") % { + 'uri': uri, + 'scheme': STORE_SCHEME + }) + LOG.error(reason) + raise exceptions.BadStoreUri(message=reason) + pieces = urllib.parse.urlparse(uri) + self.scheme = pieces.scheme + self._parse_attrs(pieces.netloc + pieces.path) + self.image_name = self.images + + +class Store(driver.Store): + """An implementation of the Azure Backend Adapter""" + + _CAPABILITIES = (capabilities.BitMasks.RW_ACCESS | + capabilities.BitMasks.DRIVER_REUSABLE) + + def __init__(self, conf): + super(Store, self).__init__(conf) + self.scheme = STORE_SCHEME + conf.register_group(config.azure_group) + conf.register_opts(config.azure_opts, group=config.azure_group) + + self.tenant_id = conf.azure.tenant_id + self.client_id = conf.azure.client_id + self.client_secret = conf.azure.client_secret + self.subscription_id = conf.azure.subscription_id + self.resource_group = conf.azure.resource_group + self._azure_client = None + LOG.info('Initialized Azure Glance Store driver') + + def get_schemes(self): + return (STORE_SCHEME, ) + + @property + def azure_client(self): + if self._azure_client is None: + self._azure_client = utils.get_compute_client( + self.tenant_id, self.client_id, self.client_secret, + self.subscription_id) + return self._azure_client + + @capabilities.check + def get(self, location, offset=0, chunk_size=None, context=None): + """Takes a `glance_store.location.Location` object that indicates + where to find the image file, and returns a tuple of generator + (for reading the image file) and image_size + :param location `glance_store.location.Location` object, supplied + from glance_store.location.get_location_from_uri() + """ + return '%s://generic' % self.scheme, self.get_size(location, context) + + def get_size(self, location, context=None): + image_name = location.store_location.image_name.split("/")[-1] + response = utils.get_image(self.azure_client, self.resource_group, + image_name) + size = response.storage_profile.os_disk.disk_size_gb + if size is None: + return 1 + return size * units.Gi + + @capabilities.check + def add(self, image_id, image_file, image_size, context=None, + verifier=None): + """Stores an image file with supplied identifier to the backend + storage system and returns a tuple containing information + about the stored image. + :param image_id: The opaque image identifier + :param image_file: The image data to write, as a file-like object + :param image_size: The size of the image data to write, in bytes + :retval: tuple of URL in backing store, bytes written, checksum + and a dictionary with storage system specific information + :raises: `glance_store.exceptions.Duplicate` if the image already + existed + """ + # Adding images is not suppported yet + raise NotImplementedError("This operation is not supported in Azure") + + @capabilities.check + def delete(self, location, context=None): + """Takes a `glance_store.location.Location` object that indicates + where to find the image file to delete + :param location: `glance_store.location.Location` object, supplied + from glance_store.location.get_location_from_uri() + :raises NotFound if image does not exist + """ + # This method works for Azure public images as we just need to delete + # entry from glance catalog. + # For Private images we will need extra handling here. + LOG.info("Delete image %s" % location.get_store_uri()) diff --git a/glance/glance_store/_drivers/azure/utils.py b/glance/glance_store/_drivers/azure/utils.py new file mode 100644 index 0000000..44a5a20 --- /dev/null +++ b/glance/glance_store/_drivers/azure/utils.py @@ -0,0 +1,49 @@ +""" +Copyright (c) 2017 Platform9 Systems Inc. +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the +License for the specific language governing permissions and limitations +under the License. +""" + +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.compute import ComputeManagementClient +from functools import partial +from oslo_log import log as logging + +LOG = logging.getLogger(__name__) + + +def get_credentials(tenant_id, client_id, client_secret): + credentials = ServicePrincipalCredentials( + client_id=client_id, secret=client_secret, tenant=tenant_id) + return credentials + + +def _get_client(tenant_id, client_id, client_secret, subscription_id, + cls=None): + """Returns Azure compute resource object for interacting with Azure API + + :param tenant_id: string, tenant_id from azure account + :param client_id: string, client_id (application id) + :param client_secret: string, secret key of application + :param subscription_id: string, unique identification id of account + :return: :class:`Resource ` object + """ + credentials = get_credentials(tenant_id, client_id, client_secret) + client = cls(credentials, subscription_id) + return client + + +get_compute_client = partial(_get_client, cls=ComputeManagementClient) + + +def get_image(compute, resource_group, name): + """Return image info from Azure + """ + return compute.images.get(resource_group, name)