From 883d0c5289b7b237fc8f3b175116836cf2ccdf3d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 6 Jan 2015 13:08:47 -0500 Subject: [PATCH] Support uploading swift objects In addition to just needing to be able to upload swift objects into containers, we should be able to sanely interact with swift on general principle. Change-Id: I844ea7a26005e72ab037fe733681058944e847fa --- requirements.txt | 1 + shade/__init__.py | 100 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 610303351..c284bfde6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ python-cinderclient python-neutronclient python-troveclient python-ironicclient +python-swiftclient diff --git a/shade/__init__.py b/shade/__init__.py index bd9cd5e0e..fbfafd1f9 100644 --- a/shade/__init__.py +++ b/shade/__init__.py @@ -14,6 +14,7 @@ import logging import operator +import subprocess import time from cinderclient import exceptions as cinder_exceptions @@ -27,6 +28,8 @@ from novaclient.v1_1 import client as nova_client from novaclient.v1_1 import floating_ips import os_client_config import pbr.version +import swiftclient.client as swift_client +import swiftclient.exceptions as swift_exceptions import troveclient.client as trove_client from troveclient import exceptions as trove_exceptions @@ -51,11 +54,12 @@ def openstack_clouds(config=None): for f in config.get_all_clouds()] -def openstack_cloud(**kwargs): +def openstack_cloud(debug=False, **kwargs): cloud_config = os_client_config.OpenStackConfig().get_one_cloud( **kwargs) return OpenStackCloud( - cloud_config.name, cloud_config.region, **cloud_config.config) + cloud_config.name, cloud_config.region, + debug=debug, **cloud_config.config) def operator_cloud(**kwargs): @@ -102,6 +106,7 @@ class OpenStackCloud(object): self._image_cache = image_cache self._flavor_cache = flavor_cache self._volume_cache = volume_cache + self._container_cache = dict() self.debug = debug @@ -111,9 +116,13 @@ class OpenStackCloud(object): self._keystone_client = None self._cinder_client = None self._trove_client = None + self._swift_client = None self.log = logging.getLogger('shade') - self.log.setLevel(logging.INFO) + log_level = logging.INFO + if self.debug: + log_level = logging.DEBUG + self.log.setLevel(log_level) self.log.addHandler(logging.StreamHandler()) def get_service_type(self, service): @@ -224,6 +233,20 @@ class OpenStackCloud(object): raise OpenStackCloudException("Error connecting to glance") return self._glance_client + @property + def swift_client(self): + if self._swift_client is None: + token = self.keystone_client.auth_token + endpoint = self.get_endpoint( + service_type=self.get_service_type('object-store')) + self._swift_client = swift_client.Connection( + preauthurl=endpoint, + preauthtoken=token, + #auth_version='2.0', + os_options=dict(region_name=self.region_name), + ) + return self._swift_client + @property def cinder_client(self): @@ -632,6 +655,77 @@ class OpenStackCloud(object): raise OpenStackCloudTimeout( "Timed out waiting for server to get deleted.") + def get_container(self, name, skip_cache=False): + if not skip_cache and name in self._container_cache: + return self._container_cache[name] + try: + container = self.swift_client.head_container(name) + self._container_cache[name] = container + except swift_exceptions.ClientException as e: + if e.http_status == 404: + return None + raise + + def create_container(self, name): + if self.get_container(name): + return + try: + self.swift_client.put_container(name) + self.get_container(name, skip_cache=True) + except swift_exceptions.ClientException as e: + raise OpenStackCloudException( + "Container creation failed: %s (%s/%s)" % ( + e.http_reason, e.http_host, e.http_path)) + + def _get_file_md5(self, filename): + return subprocess.check_output(['md5sum', filename]).split()[0] + + def _is_object_stale(self, container, name, filename, file_md5): + + try: + metadata = self.get_object_metadata(container, name) + except OpenStackCloudException: + self.log.debug( + "swift stale check, no object: {container}/{name}".format( + container=container, name=name)) + return True + if file_md5 is None: + file_md5 = self._get_file_md5(filename) + if file_md5 != metadata['etag']: + self.log.debug( + "swift md5 mismatch: {filename}!={container}/{name}".format( + filename=filename, container=container, name=name)) + return True + self.log.debug( + "swift object up to date: {container}/{name}".format( + container=container, name=name)) + return False + + def create_object( + self, container, name, filename=None, md5=None, headers=None): + if not filename: + filename = name + + if self._is_object_stale(container, name, filename, md5): + + self.create_container(container) + + with open(filename, 'r') as fileobj: + self.log.debug( + "swift uploading {filename} to {container}/{name}".format( + filename=filename, container=container, name=name)) + self.swift_client.put_object(container, name, contents=fileobj) + if headers: + self.swift_client.post_object(container, name, headers=headers) + + def get_object_metadata(self, container, name): + try: + return self.swift_client.head_object(container, name) + except swift_exceptions.ClientException as e: + raise OpenStackCloudException( + "Object metadata fetch failed: %s (%s/%s)" % ( + e.http_reason, e.http_host, e.http_path)) + class OperatorCloud(OpenStackCloud):