Subject: Add Licenses. Pep8 fixes.

added licenses to tests. added __init__.py's where they were needed.
pep8 fixes

Change-Id: I5404685c098b75d862dfb9c64482d2685e0b645b
This commit is contained in:
Carlos 'sn1p3r' Martinez 2013-05-16 15:22:35 -05:00
parent 9e534e0e19
commit 57466ac8ab
10 changed files with 554 additions and 65 deletions

View File

@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -0,0 +1,15 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -0,0 +1,41 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import hashlib
def get_md5_hash(data, block_size_multiplier=1):
"""
returns an md5 sum. data is a string or file pointer.
block size is 512 (md5 msg length).
"""
hash_ = None
default_block_size = 2 ** 9
block_size = block_size_multiplier * default_block_size
md5 = hashlib.md5()
if type(data) is file:
while True:
read_data = data.read(block_size)
if not read_data:
break
md5.update(read_data)
data.close()
else:
md5.update(str(data))
hash_ = md5.hexdigest()
return hash_

View File

@ -0,0 +1,50 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from uuid import uuid4
def get_random_string(prefix=None, suffix=None, size=8):
"""
Return exactly size bytes worth of base_text as a string
surrounded by any defined pre or suf-fixes
"""
body = ''
base_text = str(uuid4()).replace('-', '0')
if size <= 0:
body = '{0}{1}'.format(prefix, suffix)
else:
extra = size % len(base_text)
if extra == 0:
body = base_text * size
if extra == size:
body = base_text[:size]
if (extra > 0) and (extra < size):
temp_len = (size / len(base_text))
base_one = base_text * temp_len
base_two = base_text[:extra]
body = '{0}{1}'.format(base_one, base_two)
if prefix is not None:
body = '{0}{1}'.format(str(prefix), str(body))
if suffix is not None:
body = '{0}{1}'.format(str(body), str(suffix))
return body

View File

@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -0,0 +1,15 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -14,3 +14,20 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
from cafe.engine.behaviors import BaseBehavior, behavior
from cloudcafe.objectstorage.objectstorage_api.config \
import ObjectStorageAPIConfig
from cloudcafe.objectstorage.objectstorage_api.client \
import ObjectStorageAPIClient
class ObjectStorageAPI_Behaviors(BaseBehavior):
def __init__(self, client=None):
self.client = client
self.cofnig = ObjectStorageAPIConfig()
@behavior(ObjectStorageAPIClient)
def create_container(self, name=None):
response = self.client.create_container(name)
if not response.ok:
raise Exception('could not create container')

View File

@ -13,40 +13,68 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import cStringIO
import datetime
import hmac
import json
import tarfile
import time
import urllib
from time import time
from hashlib import sha1
from cafe.engine.clients.rest import RestClient
from cloudcafe.objectstorage.objectstorage_api.models.responses \
import AccountContainersList, ContainerObjectsList
class ObjectStorageClient(RestClient):
def _deserialize(response_entity_type):
"""
Auto-deserializes the response from any decorated client method call
that has a 'format' key in it's 'params' dictionary argument, where
'format' value is either 'json' or 'xml'.
Deserializes the response into response_entity_type domain object
response_entity_type must be a Domain Object with a <format>_to_obj()
classmethod defined for every supported format or this won't work.
"""
def decorator(f):
def wrapper(*args, **kwargs):
response = f(*args, **kwargs)
response.request.__dict__['entity'] = None
response.__dict__['entity'] = None
deserialize_format = None
if isinstance(kwargs, dict):
if isinstance(kwargs.get('params'), dict):
deserialize_format = kwargs['params'].get('format')
if deserialize_format is not None:
response.__dict__['entity'] = \
response_entity_type.deserialize(
response.content, deserialize_format)
return response
return wrapper
return decorator
class ObjectStorageAPIClient(RestClient):
def __init__(self, storage_url, auth_token, base_container_name=None,
base_object_name=None):
super(ObjectStorageClient, self).__init__()
super(ObjectStorageAPIClient, self).__init__()
self.storage_url = storage_url
self.auth_token = auth_token
self.base_container_name = base_container_name
self.base_object_name = base_object_name
self.base_container_name = base_container_name or ''
self.base_object_name = base_object_name or ''
self.default_headers['X-Auth-Token'] = self.auth_token
def __add_object_metadata_to_headers(self, metadata=None, headers=None):
"""
Call to __build_metadata specifically for object headers
"""
return self.__build_metadata('X-Object-Meta-', metadata, headers)
def __add_container_metadata_to_headers(self, metadata=None, headers=None):
"""
Call to __build_metadata specifically for container headers
"""
return self.__build_metadata('X-Container-Meta-', metadata, headers)
def __add_account_metadata_to_headers(self, metadata=None, headers=None):
@ -80,11 +108,11 @@ class ObjectStorageClient(RestClient):
for key in metadata:
try:
meta_key = ''.join([prefix, key])
meta_key = '{0}{1}'.format(prefix, key)
except TypeError as e:
self.client_log.error(
'Non-string prefix OR metadata dict value was passed '
'to __build_metadata() in object_storage_client.py')
'to __build_metadata() in object_client.py')
self.client_log.exception(e)
raise
except:
@ -93,82 +121,145 @@ class ObjectStorageClient(RestClient):
return dict(metadata_headers, **headers)
def create_container(self, container_name, metadata=None, headers=None,
requestslib_kwargs=None):
#Account-------------------------------------------------------------------
headers = self.__add_container_metadata_to_headers(metadata, headers)
url = '{0}/{1}'.format(self.storage_url, container_name)
response = self.request(
'PUT',
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
def retrieve_account_metadata(self, requestslib_kwargs=None):
"""4.1.1 View Account Details"""
response = self.head(
self.storage_url,
requestslib_kwargs=requestslib_kwargs)
return response
def create_storage_object(self, container_name, object_name, data=None,
metadata=None, headers=None,
requestslib_kwargs=None):
@_deserialize(AccountContainersList)
def list_containers(self, headers=None, params=None,
requestslib_kwargs=None):
"""
Creates a storage object in a container via PUT
Optionally adds 'X-Object-Metadata-' prefix to any key in the
metadata dictionary, and then adds that metadata to the headers
dictionary.
Lists all containers for the account.
If the 'format' variable is passed as part of the 'params'
dictionary, an object representing the deserialized version of
that format (either xml or json) will be appended to the response
as the 'entity' attribute. (ie, response.entity)
"""
headers = self.__add_object_metadata_to_headers(metadata, headers)
response = self.get(
self.storage_url,
headers=headers,
params=params,
requestslib_kwargs=requestslib_kwargs)
url = '{0}/{1}/{2}'.format(
self.storage_url,
container_name,
object_name)
return response
response = self.request(
'PUT',
url,
headers=headers,
data=data,
requestslib_kwargs=requestslib_kwargs)
#Container-----------------------------------------------------------------
def get_container_metadata(self, container_name, headers=None,
requestslib_kwargs=None):
"""4.2.1 View Container Details"""
url = '{0}/{1}'.format(self.storage_url, container_name)
response = self.head(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def create_container(self, container_name, metadata=None, headers=None,
requestslib_kwargs=None):
url = '{0}/{1}'.format(self.storage_url, container_name)
headers = self.__add_container_metadata_to_headers(metadata, headers)
response = self.put(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def update_container(self, container_name, headers=None,
requestslib_kwargs=None):
url = '{0}/{1}'.format(self.storage_url, container_name)
response = self.put(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def delete_container(self, container_name, headers=None,
requestslib_kwargs=None):
url = '{0}/{1}'.format(self.storage_url, container_name)
response = self.request(
'DELETE',
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
response = self.delete(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def delete_storage_object(self, container_name, object_name, headers=None,
def set_container_metadata(self, container_name, metadata, headers=None,
requestslib_kwargs=None):
url = '{0}/{1}'.format(self.storage_url, container_name)
headers = self.__add_container_metadata_to_headers(metadata, headers)
response = self.post(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def get_container_options(self, container_name, headers=None,
requestslib_kwargs=None):
"""4.2.5 CORS Container Headers"""
url = '{0}/{1}'.format(self.storage_url, container_name)
url = '{0}/{1}/{2}'.format(
self.storage_url,
container_name,
object_name)
response = self.request(
'DELETE',
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
response = self.options(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
@_deserialize(ContainerObjectsList)
def list_objects(self, container_name, headers=None, params=None,
requestslib_kwargs=None):
"""
Lists all objects in the specified container.
If the 'format' variable is passed as part of the 'params'
dictionary, an object representing the deserialized version of
that format (either xml or json) will be appended to the response
as the 'entity' attribute. (ie, response.entity)
"""
url = '{0}/{1}'.format(self.storage_url, container_name)
response = self.get(
url,
headers=headers,
params=params,
requestslib_kwargs=requestslib_kwargs)
return response
def get_object_count(self, container_name):
"""
Returns the number of objects in a container.
"""
response = self.get_container_metadata(container_name)
obj_count = int(response.headers['x-container-object-count'])
return obj_count
def _purge_container(self, container_name):
params = {'format': 'json'}
r = self.list_objects(container_name, params=params)
response = self.list_objects(container_name, params=params)
try:
json_data = json.loads(r.content)
json_data = json.loads(response.content)
for entry in json_data:
self.delete_storage_object(container_name, entry['name'])
self.delete_object(container_name, entry['name'])
except Exception:
pass
@ -177,3 +268,152 @@ class ObjectStorageClient(RestClient):
def force_delete_containers(self, container_list):
for container_name in container_list:
return self._purge_container(container_name)
#Storage Object------------------------------------------------------------
def get_object(self, container_name, object_name, headers=None,
prefetch=True, requestslib_kwargs=None):
"""
optional headers
If-Match
If-None-Match
If-Modified-Since
If-Unmodified-Since
Range
If-Match and If-None-Match check the ETag header
200 on 'If' header success
If none of the entity tags match, or if "*" is given and no current
entity exists, the server MUST NOT perform the requested method, and
MUST return a 412 (Precondition Failed) response.
206 (Partial content) for successful range request
If the entity tag does not match, then the server SHOULD
return the entire entity using a 200 (OK) response
see RFC2616
If prefetch=False, body download is delayed until response.content is
accessed either directly, via response.iter_content() or .iter_lines()
"""
url = '{0}/{1}/{2}'.format(
self.storage_url,
container_name,
object_name)
if requestslib_kwargs is None:
requestslib_kwargs = {}
if requestslib_kwargs.get('prefetch') is None:
requestslib_kwargs['prefetch'] = prefetch
response = self.get(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def create_object(self, container_name, object_name, data=None,
metadata=None, headers=None, requestslib_kwargs=None):
"""
Creates a storage object in a container via PUT
Optionally adds 'X-Object-Metadata-' prefix to any key in the
metadata dictionary, and then adds that metadata to the headers
dictionary.
"""
url = '{0}/{1}/{2}'.format(
self.storage_url,
container_name,
object_name)
hdrs = self.__add_object_metadata_to_headers(metadata, headers)
response = self.put(
url,
headers=hdrs,
data=data,
requestslib_kwargs=requestslib_kwargs)
return response
def copy_object(self, container_name, object_name, headers=None):
url = '{0}/{1}/{2}'.format(
self.storage_url,
container_name,
object_name)
hdrs = {}
hdrs['X-Auth-Token'] = self.auth_token
if headers is not None:
if 'X-Copy-From' in headers and 'Content-Length' in headers:
method = 'PUT'
hdrs['X-Copy-From'] = headers['X-Copy-From']
hdrs['Content-Length'] = headers['Content-Length']
elif 'Destination' in headers:
method = 'COPY'
hdrs['Destination'] = headers['Destination']
else:
return None
response = self.request(method=method, url=url, headers=hdrs)
return response
def delete_object(self, container_name, object_name, headers=None,
requestslib_kwargs=None):
url = '{0}/{1}/{2}'.format(
self.storage_url,
container_name,
object_name)
response = self.delete(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def get_object_metadata(self, container_name, object_name,
headers=None, requestslib_kwargs=None):
url = '{0}/{1}/{2}'.format(
self.storage_url,
container_name,
object_name)
response = self.head(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def set_object_metadata(self, container_name, object_name, metadata,
headers=None, requestslib_kwargs=None):
url = '{0}/{1}/{2}'.format(
self.storage_url,
container_name,
object_name)
headers = self.__add_object_metadata_to_headers(metadata, headers)
response = self.post(
url,
headers=headers,
requestslib_kwargs=requestslib_kwargs)
return response
def set_temp_url_key(self, headers=None, requestslib_kwargs=None):
return self.post(self.storage_url, headers=headers)
def create_temp_url(self, method, container, obj, seconds, key,
headers=None, requestslib_kwargs=None):
method = method.upper()
base_url = '{0}/{1}/{2}'.format(self.storage_url, container, obj)
account_hash = self.storage_url.split('/v1/')[1]
object_path = '/v1/{0}/{1}/{2}'.format(account_hash, container, obj)
seconds = int(seconds)
expires = int(time() + seconds)
hmac_body = '{0}\n{1}\n{2}'.format(method, expires, object_path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
return {'target_url': base_url, 'signature': sig, 'expires': expires}

View File

@ -0,0 +1,15 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -0,0 +1,98 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import json
from xml.etree import ElementTree
from cafe.engine.models.base import AutoMarshallingModel
class AccountContainersList(AutoMarshallingModel):
pass
class _Container(object):
def __init__(self, name=None, count=None, bytes=None):
self.name = None
self.count = None
self.bytes = None
def __init__(self):
'''This is a deserializing object only'''
pass
@classmethod
def _xml_to_obj(cls, serialized_str):
ret = []
root = ElementTree.fromstring(serialized_str)
setattr(cls, 'name', root.attrib['name'])
for child in root:
container_dict = {}
for sub_child in child:
container_dict[sub_child.tag] = sub_child.text
ret.append(cls._StorageObject(**container_dict))
return ret
@classmethod
def _json_to_obj(cls, serialized_str):
ret = []
data = json.loads(serialized_str)
for container in data:
ret.append(
cls._Container(
name=container.get('name'),
bytes=container.get('bytes'),
count=container.get('count')))
return ret
class ContainerObjectsList(AutoMarshallingModel):
#TODO: make this not use *args and **kwargs
class _StorageObject(dict):
def __init__(self, *args, **kwargs):
self.name = kwargs.get('name', None)
self.bytes = kwargs.get('bytes', None)
self.hash = kwargs.get('hash', None)
self.last_modified = kwargs.get('last_modified', None)
self.content_type = kwargs.get('content_type', None)
def __init__(self):
'''This is a deserializing object only'''
pass
@classmethod
def _xml_to_obj(cls, serialized_str):
ret = []
root = ElementTree.fromstring(serialized_str)
setattr(cls, 'name', root.attrib['name'])
for child in root:
storage_object_dict = {}
for sub_child in child:
storage_object_dict[sub_child.tag] = sub_child.text
ret.append(cls._StorageObject(**storage_object_dict))
return ret
@classmethod
def _json_to_obj(cls, serialized_str):
ret = []
data = json.loads(serialized_str)
for storage_object in data:
storage_obj = cls._StorageObject(
name=storage_object.get('name'),
bytes=storage_object.get('bytes'),
hash=storage_object.get('hash'))
ret.append(
storage_obj,
last_modified=storage_object.get('last_modified'),
content_type=storage_object.get('content_type'))
return ret