openstack/storlets
Code Issues Proposed changes

Use unique container for objects

Browse Source 
any other system containers (storlets, dependencies, etc) are still same

Change-Id: I002cf5aa7fb0fcf83dee0c955c1d4d24294f2776
This commit is contained in:
Kota Tsuyuzaki
2017-03-07 18:39:55 -08:00
parent e6aec4dda3
commit 70fe091e7a
24 changed files with 103 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
StorletSamples/java/CompressStorlet/build.xml
View File

@@ -24,6 +24,7 @@
classpath="../../../src/java/SCommon/bin/SCommon.jar" classpath="../../../src/java/SCommon/bin/SCommon.jar"
includeantruntime="false" /> includeantruntime="false" />
</target> </target>
<!--TODO: why do we need input.txt from storlets html?-->
<target name="text"> <target name="text">
<exec dir="bin" executable="wget"> <exec dir="bin" executable="wget">
<arg line="https://github.com/openstack/storlets -O input.txt" /> <arg line="https://github.com/openstack/storlets -O input.txt" />

45
tests/functional/__init__.py
View File

@@ -15,20 +15,34 @@
# limitations under the License. # limitations under the License.
import unittest import unittest
import uuid
from swiftclient import client as swiftclient from swiftclient import client as swiftclient
from storlets.tools.cluster_config_parser import ClusterConfig from storlets.tools.cluster_config_parser import ClusterConfig
from storlets.tools.utils import deploy_storlet, get_admin_auth, put_local_file from storlets.tools.utils import deploy_storlet, get_admin_auth, put_local_file
import os
CONFIG_FILE = '../../cluster_config.json' CONFIG_DIR = os.environ.get('CLUSTER_CONF_DIR', os.getcwd())
PATH_TO_STORLETS = '../../StorletSamples' CONFIG_FILE = os.path.join(CONFIG_DIR, 'cluster_config.json')
PATH_TO_STORLETS = os.environ.get(
'STORLET_SAMPLE_PATH',
# assuming, current working dir is at top of storlet repo
os.path.join(os.getcwd(), 'StorletSamples'))
CONSOLE_TIMEOUT = 2 CONSOLE_TIMEOUT = 2
class StorletBaseFunctionalTest(unittest.TestCase): class StorletBaseFunctionalTest(unittest.TestCase):
def setUp(self): def setUp(self):
self.conf_file = CONFIG_FILE self.conf_file = CONFIG_FILE
self.conf = ClusterConfig(CONFIG_FILE) try:
self.conf = ClusterConfig(CONFIG_FILE)
except IOError:
self.fail('cluster_config.json not found in %s. '
'Please check your testing environment.' % CONFIG_DIR)
self.url, self.token = get_admin_auth(self.conf) self.url, self.token = get_admin_auth(self.conf)
# TODO(kota_): do we need to call setUp() when inheriting TestCase
# directly? AFAIK, no setUp method in the class...
super(StorletBaseFunctionalTest, self).setUp() super(StorletBaseFunctionalTest, self).setUp()
@@ -42,10 +56,26 @@ class StorletFunctionalTest(StorletBaseFunctionalTest):
status = response.get('status') status = response.get('status')
assert (status >= 200 or status < 300) assert (status >= 200 or status < 300)
def cleanup_container(self, container):
# list all objects inside the container
_, objects = swiftclient.get_container(
self.url, self.token, container, full_listing=True)
# delete all objects inside the container
# N.B. this cleanup could run in parallel but currently we have a few
# objects in the user testing container so that, currently this does
# as sequential simply
for obj_dict in objects:
swiftclient.delete_object(
self.url, self.token, container, obj_dict['name'])
swiftclient.get_container(self.url, self.token, container)
# delete the container
swiftclient.delete_container(self.url, self.token, container)
def setUp(self, language, path_to_bundle, def setUp(self, language, path_to_bundle,
storlet_dir, storlet_dir,
storlet_name, storlet_main, storlet_name, storlet_main, storlet_file,
container, storlet_file,
dep_names, headers): dep_names, headers):
super(StorletFunctionalTest, self).setUp() super(StorletFunctionalTest, self).setUp()
self.storlet_dir = storlet_dir self.storlet_dir = storlet_dir
@@ -53,7 +83,7 @@ class StorletFunctionalTest(StorletBaseFunctionalTest):
self.storlet_main = storlet_main self.storlet_main = storlet_main
self.dep_names = dep_names self.dep_names = dep_names
self.path_to_bundle = path_to_bundle self.path_to_bundle = path_to_bundle
self.container = container self.container = 'container-%s' % uuid.uuid4()
self.storlet_file = storlet_file self.storlet_file = storlet_file
self.headers = headers or {} self.headers = headers or {}
self.acct = self.url.split('/')[4] self.acct = self.url.split('/')[4]
@@ -74,3 +104,6 @@ class StorletFunctionalTest(StorletBaseFunctionalTest):
self.path_to_bundle, self.path_to_bundle,
self.storlet_file, self.storlet_file,
self.headers) self.headers)
def tearDown(self):
self.cleanup_container(self.container)

3
tests/functional/java/__init__.py
View File

@@ -21,7 +21,7 @@ BIN_DIR = 'bin'
class StorletJavaFunctionalTest(StorletFunctionalTest): class StorletJavaFunctionalTest(StorletFunctionalTest):
def setUp(self, storlet_dir, storlet_name, storlet_main, def setUp(self, storlet_dir, storlet_name, storlet_main,
container, storlet_file, dep_names=None, headers=None): storlet_file, dep_names=None, headers=None):
storlet_dir = os.path.join('java', storlet_dir) storlet_dir = os.path.join('java', storlet_dir)
path_to_bundle = os.path.join(PATH_TO_STORLETS, storlet_dir, path_to_bundle = os.path.join(PATH_TO_STORLETS, storlet_dir,
BIN_DIR) BIN_DIR)
@@ -30,7 +30,6 @@ class StorletJavaFunctionalTest(StorletFunctionalTest):
storlet_dir, storlet_dir,
storlet_name, storlet_name,
storlet_main, storlet_main,
container,
storlet_file, storlet_file,
dep_names, dep_names,
headers) headers)

7
tests/functional/java/test_SLO.py
View File

@@ -47,7 +47,7 @@ class TestSLO(StorletJavaFunctionalTest):
super(TestSLO, self).setUp('IdentityStorlet', super(TestSLO, self).setUp('IdentityStorlet',
'identitystorlet-1.0.jar', 'identitystorlet-1.0.jar',
main_class, main_class,
'myobjects', '') '')
for cont in ('container1', 'container2', 'container3'): for cont in ('container1', 'container2', 'container3'):
self.create_container(cont) self.create_container(cont)
@@ -57,6 +57,7 @@ class TestSLO(StorletJavaFunctionalTest):
def tearDown(self): def tearDown(self):
delete_local_chunks() delete_local_chunks()
super(TestSLO, self).tearDown()
def get_SLO(self): def get_SLO(self):
response = dict() response = dict()
@@ -97,7 +98,7 @@ class TestSLO(StorletJavaFunctionalTest):
headers = response.get('headers') headers = response.get('headers')
segment = dict() segment = dict()
segment['path'] = 'myobjects/%s' % oname segment['path'] = '%s/%s' % (self.container, oname)
segment['size_bytes'] = 1048576 segment['size_bytes'] = 1048576
segment['etag'] = headers['etag'] segment['etag'] = headers['etag']
assembly.append(segment) assembly.append(segment)
@@ -142,7 +143,7 @@ class TestSLO(StorletJavaFunctionalTest):
headers.update(self.additional_headers) headers.update(self.additional_headers)
response = dict() response = dict()
headers, body = c.get_object(self.url, self.token, headers, body = c.get_object(self.url, self.token,
'myobjects', 'assembly', self.container, 'assembly',
query_string=None, query_string=None,
response_dict=response, response_dict=response,
resp_chunk_size=1048576, resp_chunk_size=1048576,

6
tests/functional/java/test_compress_storlet.py
View File

@@ -27,7 +27,6 @@ class TestCompressStorlet(StorletJavaFunctionalTest):
super(TestCompressStorlet, self).setUp('CompressStorlet', super(TestCompressStorlet, self).setUp('CompressStorlet',
'compressstorlet-1.0.jar', 'compressstorlet-1.0.jar',
main_class, main_class,
'myobjects',
'input.txt') 'input.txt')
def test_put(self): def test_put(self):
@@ -35,9 +34,8 @@ class TestCompressStorlet(StorletJavaFunctionalTest):
headers.update(self.additional_headers) headers.update(self.additional_headers)
querystring = "action=compress" querystring = "action=compress"
with open('../../StorletSamples/java/CompressStorlet/bin/input.txt', # simply set 1KB string data to compress
'r') as myfile: data = 'A' * 1024
data = myfile.read()
response = dict() response = dict()
c.put_object(self.url, self.token, self.container, self.storlet_file, c.put_object(self.url, self.token, self.container, self.storlet_file,

1
tests/functional/java/test_csvstorlet.py
View File

@@ -26,7 +26,6 @@ class TestCsvStorlet(StorletJavaFunctionalTest):
super(TestCsvStorlet, self).setUp('CsvStorlet', super(TestCsvStorlet, self).setUp('CsvStorlet',
'csvstorlet-1.0.jar', 'csvstorlet-1.0.jar',
main_class, main_class,
'myobjects',
'meter-1MB.csv') 'meter-1MB.csv')
def invoke_storlet(self, start, end, def invoke_storlet(self, start, end,

3
tests/functional/java/test_deploy_storlet.py
View File

@@ -20,12 +20,13 @@ from tests.functional import StorletBaseFunctionalTest, PATH_TO_STORLETS, \
CONSOLE_TIMEOUT CONSOLE_TIMEOUT
from tests.functional.java import BIN_DIR from tests.functional.java import BIN_DIR
import unittest import unittest
from storlets.tools import deploy_storlet
class TestDeployStorlet(StorletBaseFunctionalTest): class TestDeployStorlet(StorletBaseFunctionalTest):
def setUp(self): def setUp(self):
super(TestDeployStorlet, self).setUp() super(TestDeployStorlet, self).setUp()
self.deploy_storlet_path = '../../storlets/tools/deploy_storlet.py' self.deploy_storlet_path = os.path.abspath(deploy_storlet.__file__)
self.execdep_storlet_path = os.path.join(PATH_TO_STORLETS, self.execdep_storlet_path = os.path.join(PATH_TO_STORLETS,
'java', 'java',
'ExecDepStorlet', 'ExecDepStorlet',

3
tests/functional/java/test_execdep_storlet.py
View File

@@ -27,7 +27,6 @@ class TestExecDepStorlet(StorletJavaFunctionalTest):
super(TestExecDepStorlet, self).setUp('ExecDepStorlet', super(TestExecDepStorlet, self).setUp('ExecDepStorlet',
'execdepstorlet-1.0.jar', 'execdepstorlet-1.0.jar',
main_class, main_class,
'myobjects',
'junk.txt', 'junk.txt',
['get42']) ['get42'])
@@ -36,7 +35,7 @@ class TestExecDepStorlet(StorletJavaFunctionalTest):
headers.update(self.additional_headers) headers.update(self.additional_headers)
resp = dict() resp = dict()
resp_headers, gf = c.get_object(self.url, self.token, resp_headers, gf = c.get_object(self.url, self.token,
'myobjects', self.container,
self.storlet_file, self.storlet_file,
response_dict=resp, response_dict=resp,
headers=headers) headers=headers)

7
tests/functional/java/test_half_storlet.py
View File

@@ -30,7 +30,6 @@ class TestHalfIdentityStorlet(StorletJavaFunctionalTest):
super(TestHalfIdentityStorlet, self).setUp('HalfStorlet', super(TestHalfIdentityStorlet, self).setUp('HalfStorlet',
'halfstorlet-1.0.jar', 'halfstorlet-1.0.jar',
main_class, main_class,
'myobjects',
'source.txt', 'source.txt',
headers=headers) headers=headers)
@@ -52,13 +51,13 @@ class TestHalfIdentityStorlet(StorletJavaFunctionalTest):
if op == 'GET': if op == 'GET':
# Get original object # Get original object
original_h, original_c = \ original_h, original_c = \
c.get_object(self.url, self.token, 'myobjects', c.get_object(self.url, self.token, self.container,
self.storlet_file, self.storlet_file,
response_dict=dict()) response_dict=dict())
# print original_headers # print original_headers
file_length = int(original_h['content-length']) file_length = int(original_h['content-length'])
processed_h, returned_c = \ processed_h, returned_c = \
c.get_object(self.url, self.token, 'myobjects', c.get_object(self.url, self.token, self.container,
self.storlet_file, self.storlet_file,
query_string=querystring, response_dict=dict(), query_string=querystring, response_dict=dict(),
headers=req_headers, resp_chunk_size=file_length) headers=req_headers, resp_chunk_size=file_length)
@@ -89,7 +88,7 @@ class TestHalfIdentityStorlet(StorletJavaFunctionalTest):
"application/octet-stream", headers, None, None, "application/octet-stream", headers, None, None,
querystring, response) querystring, response)
resp_headers, saved_content = c.get_object(self.url, self.token, resp_headers, saved_content = c.get_object(self.url, self.token,
'myobjects', self.container,
'half_random_source', 'half_random_source',
response_dict=dict()) response_dict=dict())

7
tests/functional/java/test_identity_storlet.py
View File

@@ -33,7 +33,6 @@ class TestIdentityStorlet(StorletJavaFunctionalTest):
super(TestIdentityStorlet, self).setUp('IdentityStorlet', super(TestIdentityStorlet, self).setUp('IdentityStorlet',
'identitystorlet-1.0.jar', 'identitystorlet-1.0.jar',
main_class, main_class,
'myobjects',
'source.txt', 'source.txt',
['get42'], ['get42'],
headers) headers)
@@ -59,7 +58,7 @@ class TestIdentityStorlet(StorletJavaFunctionalTest):
if op == 'GET': if op == 'GET':
# Get original object # Get original object
original_h, original_c = c.get_object(self.url, self.token, original_h, original_c = c.get_object(self.url, self.token,
'myobjects', self.container,
self.storlet_file, self.storlet_file,
response_dict=dict()) response_dict=dict())
# print original_headers # print original_headers
@@ -100,7 +99,7 @@ class TestIdentityStorlet(StorletJavaFunctionalTest):
"application/octet-stream", headers, None, None, "application/octet-stream", headers, None, None,
querystring, response) querystring, response)
resp_headers, saved_c = c.get_object(self.url, self.token, resp_headers, saved_c = c.get_object(self.url, self.token,
'myobjects', self.container,
'identity_random_source', 'identity_random_source',
response_dict=dict()) response_dict=dict())
@@ -178,7 +177,7 @@ class TestIdentityStorlet(StorletJavaFunctionalTest):
headers = {'X-Run-Storlet': self.storlet_name, headers = {'X-Run-Storlet': self.storlet_name,
'X-Storlet-Range': srange} 'X-Storlet-Range': srange}
junk, content = c.get_object(self.url, self.token, junk, content = c.get_object(self.url, self.token,
'myobjects', self.container,
'small', 'small',
headers=headers, headers=headers,
response_dict=dict()) response_dict=dict())

3
tests/functional/java/test_metadata_storlet.py
View File

@@ -38,7 +38,6 @@ class TestMetadataStorlet(StorletJavaFunctionalTest):
super(TestMetadataStorlet, self).setUp('TestMetadataStorlet', super(TestMetadataStorlet, self).setUp('TestMetadataStorlet',
'testmetadatastorlet-1.0.jar', 'testmetadatastorlet-1.0.jar',
main_class, main_class,
'myobjects',
'source.txt', 'source.txt',
headers=headers) headers=headers)
@@ -55,7 +54,7 @@ class TestMetadataStorlet(StorletJavaFunctionalTest):
headers.update(self.additional_headers) headers.update(self.additional_headers)
original_headers, original_content = \ original_headers, original_content = \
c.get_object(self.url, self.token, c.get_object(self.url, self.token,
'myobjects', self.storlet_file, self.container, self.storlet_file,
response_dict=dict(), headers=headers) response_dict=dict(), headers=headers)
self.assertEqual(original_headers['X-Object-Meta-key1'.lower()], '1') self.assertEqual(original_headers['X-Object-Meta-key1'.lower()], '1')
self.assertEqual(original_headers['X-Object-Meta-key2'.lower()], '2') self.assertEqual(original_headers['X-Object-Meta-key2'.lower()], '2')

1
tests/functional/java/test_multiinput_storlet.py
View File

@@ -26,7 +26,6 @@ class TestMultiInputStorlet(StorletJavaFunctionalTest):
storlet_dir='MultiInputStorlet', storlet_dir='MultiInputStorlet',
storlet_name='multiinputstorlet-1.0.jar', storlet_name='multiinputstorlet-1.0.jar',
storlet_main='org.openstack.storlet.multiinput.MultiInputStorlet', storlet_main='org.openstack.storlet.multiinput.MultiInputStorlet',
container='myobjects',
storlet_file=None, storlet_file=None,
headers={}) headers={})

1
tests/functional/java/test_partitions_identity_storlet.py
View File

@@ -61,7 +61,6 @@ class TestPartitionsIdentityStorlet(StorletJavaFunctionalTest):
'PartitionsIdentityStorlet', 'PartitionsIdentityStorlet',
'partitionsidentitystorlet-1.0.jar', 'partitionsidentitystorlet-1.0.jar',
main_class, main_class,
'myobjects',
'records.txt') 'records.txt')
def invoke_storlet(self, start, end, first_partition, max_record_line): def invoke_storlet(self, start, end, first_partition, max_record_line):

20
tests/functional/java/test_test_storlet.py
View File

@@ -42,7 +42,6 @@ class TestTestStorlet(StorletJavaFunctionalTest):
super(TestTestStorlet, self).setUp('TestStorlet', super(TestTestStorlet, self).setUp('TestStorlet',
'test-10.jar', 'test-10.jar',
main_class, main_class,
'myobjects',
'') '')
self.member_url, self.member_token = get_member_auth(self.conf) self.member_url, self.member_token = get_member_auth(self.conf)
@@ -53,13 +52,6 @@ class TestTestStorlet(StorletJavaFunctionalTest):
'test_object', 'test_object',
'some content') 'some content')
def tearDown(self):
headers = {'X-Container-Read': ''}
swift_client.post_container(self.url,
self.token,
'myobjects',
headers)
def invokeTestStorlet(self, op, withlog=False): def invokeTestStorlet(self, op, withlog=False):
headers = {'X-Run-Storlet': self.storlet_name} headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers) headers.update(self.additional_headers)
@@ -70,7 +62,7 @@ class TestTestStorlet(StorletJavaFunctionalTest):
resp_dict = dict() resp_dict = dict()
try: try:
resp_headers, gf = swift_client.get_object(self.url, self.token, resp_headers, gf = swift_client.get_object(self.url, self.token,
'myobjects', self.container,
'test_object', 'test_object',
None, None, params, None, None, params,
resp_dict, headers) resp_dict, headers)
@@ -135,7 +127,7 @@ class TestTestStorlet(StorletJavaFunctionalTest):
exc_pattern = '^.*403 Forbidden.*$' exc_pattern = '^.*403 Forbidden.*$'
with self.assertRaisesRegexp(ClientException, exc_pattern): with self.assertRaisesRegexp(ClientException, exc_pattern):
swift_client.get_object(self.member_url, self.member_token, swift_client.get_object(self.member_url, self.member_token,
'myobjects', 'test_object', self.container, 'test_object',
headers=headers) headers=headers)
def test_storlet_acl_get_success(self): def test_storlet_acl_get_success(self):
@@ -144,24 +136,24 @@ class TestTestStorlet(StorletJavaFunctionalTest):
exc_pattern = '^.*403 Forbidden.*$' exc_pattern = '^.*403 Forbidden.*$'
with self.assertRaisesRegexp(ClientException, exc_pattern): with self.assertRaisesRegexp(ClientException, exc_pattern):
swift_client.get_object(self.member_url, self.member_token, swift_client.get_object(self.member_url, self.member_token,
'myobjects', 'test_object', self.container, 'test_object',
headers=headers) headers=headers)
headers = {'X-Storlet-Container-Read': self.conf.member_user, headers = {'X-Storlet-Container-Read': self.conf.member_user,
'X-Storlet-Name': self.storlet_name} 'X-Storlet-Name': self.storlet_name}
swift_client.post_container(self.url, swift_client.post_container(self.url,
self.token, self.token,
'myobjects', self.container,
headers) headers)
swift_client.head_container(self.url, swift_client.head_container(self.url,
self.token, self.token,
'myobjects') self.container)
headers = {'X-Run-Storlet': self.storlet_name} headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers) headers.update(self.additional_headers)
resp_dict = dict() resp_dict = dict()
swift_client.get_object(self.member_url, swift_client.get_object(self.member_url,
self.member_token, self.member_token,
'myobjects', 'test_object', self.container, 'test_object',
response_dict=resp_dict, response_dict=resp_dict,
headers=headers) headers=headers)
self.assertEqual(resp_dict['status'], 200) self.assertEqual(resp_dict['status'], 200)

23
tests/functional/java/test_thumbnail_storlet.py
View File

@@ -28,7 +28,6 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
super(TestThumbnailStorlet, self).setUp('ThumbnailStorlet', super(TestThumbnailStorlet, self).setUp('ThumbnailStorlet',
'thumbnail-1.0.jar', 'thumbnail-1.0.jar',
main_class, main_class,
'myobjects',
'sample.jpg') 'sample.jpg')
def invoke_storlet_on_get(self): def invoke_storlet_on_get(self):
@@ -36,7 +35,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
headers.update(self.additional_headers) headers.update(self.additional_headers)
resp = dict() resp = dict()
resp_headers, gf = c.get_object(self.url, self.token, resp_headers, gf = c.get_object(self.url, self.token,
'myobjects', self.container,
self.storlet_file, self.storlet_file,
response_dict=resp, response_dict=resp,
headers=headers) headers=headers)
@@ -52,7 +51,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
source_file = '%s/%s' % (self.path_to_bundle, self.storlet_file) source_file = '%s/%s' % (self.path_to_bundle, self.storlet_file)
with open(source_file, 'r') as f: with open(source_file, 'r') as f:
c.put_object(self.url, self.token, c.put_object(self.url, self.token,
'myobjects', 'gen_thumb_on_put.jpg', f, self.container, 'gen_thumb_on_put.jpg', f,
headers=headers, headers=headers,
response_dict=resp) response_dict=resp)
@@ -60,16 +59,17 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
self.assertIn(status, [201, 202]) self.assertIn(status, [201, 202])
headers = c.head_object(self.url, self.token, headers = c.head_object(self.url, self.token,
'myobjects', 'gen_thumb_on_put.jpg') self.container, 'gen_thumb_on_put.jpg')
self.assertEqual(headers['content-length'], '49032') self.assertEqual(headers['content-length'], '49032')
def invoke_storlet_on_copy_from(self): def invoke_storlet_on_copy_from(self):
headers = {'X-Run-Storlet': self.storlet_name, headers = {'X-Run-Storlet': self.storlet_name,
'X-Copy-From': 'myobjects/%s' % self.storlet_file} 'X-Copy-From': '%s/%s' %
(self.container, self.storlet_file)}
headers.update(self.additional_headers) headers.update(self.additional_headers)
resp = dict() resp = dict()
c.put_object(self.url, self.token, c.put_object(self.url, self.token,
'myobjects', 'gen_thumb_on_copy.jpg', '', self.container, 'gen_thumb_on_copy.jpg', '',
headers=headers, headers=headers,
response_dict=resp) response_dict=resp)
@@ -77,21 +77,22 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
self.assertIn(status, [201, 202]) self.assertIn(status, [201, 202])
rh = resp['headers'] rh = resp['headers']
self.assertEqual(rh['x-storlet-generated-from'], self.assertEqual(rh['x-storlet-generated-from'],
'myobjects/%s' % self.storlet_file) '%s/%s' %
(self.container, self.storlet_file))
self.assertEqual(rh['x-storlet-generated-from-account'], self.assertEqual(rh['x-storlet-generated-from-account'],
self.acct) self.acct)
self.assertIn('x-storlet-generated-from-last-modified', rh) self.assertIn('x-storlet-generated-from-last-modified', rh)
headers = c.head_object(self.url, self.token, headers = c.head_object(self.url, self.token,
'myobjects', 'gen_thumb_on_copy.jpg') self.container, 'gen_thumb_on_copy.jpg')
self.assertEqual(headers['content-length'], '49032') self.assertEqual(headers['content-length'], '49032')
def invoke_storlet_on_copy_dest(self): def invoke_storlet_on_copy_dest(self):
# No COPY in swiftclient. Using urllib instead... # No COPY in swiftclient. Using urllib instead...
url = '%s/%s/%s' % (self.url, 'myobjects', self.storlet_file) url = '%s/%s/%s' % (self.url, self.container, self.storlet_file)
headers = {'X-Auth-Token': self.token, headers = {'X-Auth-Token': self.token,
'X-Run-Storlet': self.storlet_name, 'X-Run-Storlet': self.storlet_name,
'Destination': 'myobjects/gen_thumb_on_copy_.jpg'} 'Destination': '%s/gen_thumb_on_copy_.jpg' % self.container}
headers.update(self.additional_headers) headers.update(self.additional_headers)
req = urllib2.Request(url, headers=headers) req = urllib2.Request(url, headers=headers)
req.get_method = lambda: 'COPY' req.get_method = lambda: 'COPY'
@@ -100,7 +101,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
self.assertIn(status, [201, 202]) self.assertIn(status, [201, 202])
headers = c.head_object(self.url, self.token, headers = c.head_object(self.url, self.token,
'myobjects', 'gen_thumb_on_copy_.jpg') self.container, 'gen_thumb_on_copy_.jpg')
self.assertEqual(headers['content-length'], '49032') self.assertEqual(headers['content-length'], '49032')
def test_get(self): def test_get(self):

3
tests/functional/python/__init__.py
View File

@@ -19,7 +19,7 @@ from tests.functional import StorletFunctionalTest, PATH_TO_STORLETS
class StorletPythonFunctionalTest(StorletFunctionalTest): class StorletPythonFunctionalTest(StorletFunctionalTest):
def setUp(self, storlet_dir, storlet_name, storlet_main, def setUp(self, storlet_dir, storlet_name, storlet_main,
container, storlet_file, dep_names=None, headers=None): storlet_file, dep_names=None, headers=None):
storlet_dir = os.path.join('python', 'storlet_samples', storlet_dir) storlet_dir = os.path.join('python', 'storlet_samples', storlet_dir)
path_to_bundle = os.path.join(PATH_TO_STORLETS, storlet_dir) path_to_bundle = os.path.join(PATH_TO_STORLETS, storlet_dir)
super(StorletPythonFunctionalTest, self).setUp('Python', super(StorletPythonFunctionalTest, self).setUp('Python',
@@ -27,7 +27,6 @@ class StorletPythonFunctionalTest(StorletFunctionalTest):
storlet_dir, storlet_dir,
storlet_name, storlet_name,
storlet_main, storlet_main,
container,
storlet_file, storlet_file,
dep_names, dep_names,
headers) headers)

5
tests/functional/python/test_SLO.py
View File

@@ -31,7 +31,6 @@ class TestSLO(StorletPythonFunctionalTest):
storlet_dir='simple', storlet_dir='simple',
storlet_name='simple.py', storlet_name='simple.py',
storlet_main='simple.SimpleStorlet', storlet_main='simple.SimpleStorlet',
container='myobjects',
storlet_file=None, storlet_file=None,
headers={}) headers={})
@@ -75,7 +74,7 @@ class TestSLO(StorletPythonFunctionalTest):
headers = response.get('headers') headers = response.get('headers')
segment = dict() segment = dict()
segment['path'] = 'myobjects/%s' % oname segment['path'] = '%s/%s' % (self.container, oname)
segment['size_bytes'] = 1024 * 1024 segment['size_bytes'] = 1024 * 1024
segment['etag'] = headers['etag'] segment['etag'] = headers['etag']
assembly.append(segment) assembly.append(segment)
@@ -111,7 +110,7 @@ class TestSLO(StorletPythonFunctionalTest):
headers.update(self.additional_headers) headers.update(self.additional_headers)
response = dict() response = dict()
headers, body = client.get_object(self.url, self.token, headers, body = client.get_object(self.url, self.token,
'myobjects', 'assembly', self.container, 'assembly',
query_string=None, query_string=None,
response_dict=response, response_dict=response,
resp_chunk_size=1024 * 1024, resp_chunk_size=1024 * 1024,

3
tests/functional/python/test_deploy_storlet.py
View File

@@ -18,12 +18,13 @@ import pexpect
from tests.functional import StorletBaseFunctionalTest, PATH_TO_STORLETS, \ from tests.functional import StorletBaseFunctionalTest, PATH_TO_STORLETS, \
CONSOLE_TIMEOUT CONSOLE_TIMEOUT
import unittest import unittest
from storlets.tools import deploy_storlet
class TestDeployStorlet(StorletBaseFunctionalTest): class TestDeployStorlet(StorletBaseFunctionalTest):
def setUp(self): def setUp(self):
super(TestDeployStorlet, self).setUp() super(TestDeployStorlet, self).setUp()
self.deploy_storlet_path = '../../storlets/tools/deploy_storlet.py' self.deploy_storlet_path = os.path.abspath(deploy_storlet.__file__)
self.execdep_storlet_path = os.path.join(PATH_TO_STORLETS, self.execdep_storlet_path = os.path.join(PATH_TO_STORLETS,
'python', 'python',
'storlet_samples', 'storlet_samples',

3
tests/functional/python/test_execdep_storlet.py
View File

@@ -26,7 +26,6 @@ class TestExecDepStorlet(StorletPythonFunctionalTest):
storlet_dir='exec_dep', storlet_dir='exec_dep',
storlet_name='exec_dep.py', storlet_name='exec_dep.py',
storlet_main='exec_dep.ExecDepStorlet', storlet_main='exec_dep.ExecDepStorlet',
container='myobjects',
storlet_file='source.txt', storlet_file='source.txt',
dep_names=['get42.sh'], dep_names=['get42.sh'],
headers={}) headers={})
@@ -37,7 +36,7 @@ class TestExecDepStorlet(StorletPythonFunctionalTest):
resp = dict() resp = dict()
resp_headers, get_text = client.get_object( resp_headers, get_text = client.get_object(
self.url, self.token, self.url, self.token,
'myobjects', self.container,
self.storlet_file, self.storlet_file,
response_dict=resp, response_dict=resp,
headers=headers) headers=headers)

5
tests/functional/python/test_execqueryheader_storlet.py
View File

@@ -27,7 +27,6 @@ class TestExecQueryHeaderStorlet(StorletPythonFunctionalTest):
storlet_dir='exec_query_header', storlet_dir='exec_query_header',
storlet_name='exec_query_header.py', storlet_name='exec_query_header.py',
storlet_main='exec_query_header.ExecQueryHeaderStorlet', storlet_main='exec_query_header.ExecQueryHeaderStorlet',
container='myobjects',
storlet_file='source.txt', storlet_file='source.txt',
headers={}) headers={})
@@ -51,7 +50,7 @@ class TestExecQueryHeaderStorlet(StorletPythonFunctionalTest):
if op == 'GET': if op == 'GET':
# Get original object # Get original object
original_h, original_c = client.get_object(self.url, self.token, original_h, original_c = client.get_object(self.url, self.token,
'myobjects', self.container,
self.storlet_file, self.storlet_file,
response_dict=dict()) response_dict=dict())
file_length = int(original_h['content-length']) file_length = int(original_h['content-length'])
@@ -81,7 +80,7 @@ class TestExecQueryHeaderStorlet(StorletPythonFunctionalTest):
"application/octet-stream", headers, None, None, "application/octet-stream", headers, None, None,
querystring, response) querystring, response)
resp_headers, saved_c = client.get_object(self.url, self.token, resp_headers, saved_c = client.get_object(self.url, self.token,
'myobjects', self.container,
'random_source', 'random_source',
response_dict=dict()) response_dict=dict())

1
tests/functional/python/test_multiinput_storlet.py
View File

@@ -27,7 +27,6 @@ class TestMultiInputStorlet(StorletPythonFunctionalTest):
storlet_dir='multi_input', storlet_dir='multi_input',
storlet_name='multi_input.py', storlet_name='multi_input.py',
storlet_main='multi_input.MultiInputStorlet', storlet_main='multi_input.MultiInputStorlet',
container='myobjects',
storlet_file=None, storlet_file=None,
headers={}) headers={})

16
tests/functional/python/test_simple_storlet.py
View File

@@ -31,7 +31,6 @@ class TestSimpleStorlet(StorletPythonFunctionalTest):
storlet_dir='simple', storlet_dir='simple',
storlet_name='simple.py', storlet_name='simple.py',
storlet_main='simple.SimpleStorlet', storlet_main='simple.SimpleStorlet',
container='myobjects',
storlet_file='source.txt') storlet_file='source.txt')
def test_get(self): def test_get(self):
@@ -103,21 +102,22 @@ class TestSimpleStorlet(StorletPythonFunctionalTest):
resp = dict() resp = dict()
objname = self.storlet_file + '-copy' objname = self.storlet_file + '-copy'
req_headers = {'X-Run-Storlet': self.storlet_name, req_headers = {'X-Run-Storlet': self.storlet_name,
'X-Copy-From': 'myobjects/%s' % self.storlet_file} 'X-Copy-From': '%s/%s' %
(self.container, self.storlet_file)}
client.put_object( client.put_object(
self.url, self.token, self.container, objname, self.url, self.token, self.container, objname,
self.content, response_dict=resp, headers=req_headers) self.content, response_dict=resp, headers=req_headers)
self.assertEqual(201, resp['status']) self.assertEqual(201, resp['status'])
resp_header = resp['headers'] resp_header = resp['headers']
self.assertEqual('myobjects/%s' % self.storlet_file, self.assertEqual('%s/%s' % (self.container, self.storlet_file),
resp_header['x-storlet-generated-from']) resp_header['x-storlet-generated-from'])
self.assertEqual(self.acct, self.assertEqual(self.acct,
resp_header['x-storlet-generated-from-account']) resp_header['x-storlet-generated-from-account'])
self.assertIn('x-storlet-generated-from-last-modified', resp_header) self.assertIn('x-storlet-generated-from-last-modified', resp_header)
headers = client.head_object(self.url, self.token, headers = client.head_object(self.url, self.token,
'myobjects', objname) self.container, objname)
self.assertEqual(str(len(self.content)), headers['content-length']) self.assertEqual(str(len(self.content)), headers['content-length'])
resp = dict() resp = dict()
@@ -128,25 +128,25 @@ class TestSimpleStorlet(StorletPythonFunctionalTest):
def test_copy_dest(self): def test_copy_dest(self):
# No COPY in swiftclient. Using urllib instead... # No COPY in swiftclient. Using urllib instead...
url = os.path.join(self.url, 'myobjects', self.storlet_file) url = os.path.join(self.url, self.container, self.storlet_file)
objname = self.storlet_file + '-copy-ex' objname = self.storlet_file + '-copy-ex'
headers = {'X-Auth-Token': self.token, headers = {'X-Auth-Token': self.token,
'X-Run-Storlet': self.storlet_name, 'X-Run-Storlet': self.storlet_name,
'Destination': 'myobjects/%s' % objname} 'Destination': '%s/%s' % (self.container, objname)}
headers.update(self.additional_headers) headers.update(self.additional_headers)
req = urllib2.Request(url, headers=headers) req = urllib2.Request(url, headers=headers)
req.get_method = lambda: 'COPY' req.get_method = lambda: 'COPY'
conn = urllib2.urlopen(req, timeout=10) conn = urllib2.urlopen(req, timeout=10)
self.assertEqual(201, conn.getcode()) self.assertEqual(201, conn.getcode())
self.assertEqual('myobjects/%s' % self.storlet_file, self.assertEqual('%s/%s' % (self.container, self.storlet_file),
conn.info()['x-storlet-generated-from']) conn.info()['x-storlet-generated-from'])
self.assertEqual(self.acct, self.assertEqual(self.acct,
conn.info()['x-storlet-generated-from-account']) conn.info()['x-storlet-generated-from-account'])
self.assertIn('x-storlet-generated-from-last-modified', conn.info()) self.assertIn('x-storlet-generated-from-last-modified', conn.info())
headers = client.head_object(self.url, self.token, headers = client.head_object(self.url, self.token,
'myobjects', objname) self.container, objname)
self.assertEqual(str(len(self.content)), headers['content-length']) self.assertEqual(str(len(self.content)), headers['content-length'])
resp = dict() resp = dict()

20
tests/functional/python/test_test_storlet.py
View File

@@ -40,7 +40,6 @@ class TestTestStorlet(StorletPythonFunctionalTest):
storlet_dir='test', storlet_dir='test',
storlet_name='test.py', storlet_name='test.py',
storlet_main='test.TestStorlet', storlet_main='test.TestStorlet',
container='myobjects',
storlet_file=None, storlet_file=None,
headers={}) headers={})
@@ -52,13 +51,6 @@ class TestTestStorlet(StorletPythonFunctionalTest):
'test_object', 'test_object',
'some content') 'some content')
def tearDown(self):
headers = {'X-Container-Read': ''}
swift_client.post_container(self.url,
self.token,
'myobjects',
headers)
def invoke_storlet(self, op, withlog=False): def invoke_storlet(self, op, withlog=False):
headers = {'X-Run-Storlet': self.storlet_name} headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers) headers.update(self.additional_headers)
@@ -69,7 +61,7 @@ class TestTestStorlet(StorletPythonFunctionalTest):
resp_dict = dict() resp_dict = dict()
try: try:
resp_headers, get_text = swift_client.get_object( resp_headers, get_text = swift_client.get_object(
self.url, self.token, 'myobjects', 'test_object', self.url, self.token, self.container, 'test_object',
None, None, params, resp_dict, headers) None, None, params, resp_dict, headers)
get_response_status = resp_dict.get('status') get_response_status = resp_dict.get('status')
@@ -124,7 +116,7 @@ class TestTestStorlet(StorletPythonFunctionalTest):
exc_pattern = '^.*403 Forbidden.*$' exc_pattern = '^.*403 Forbidden.*$'
with self.assertRaisesRegexp(ClientException, exc_pattern): with self.assertRaisesRegexp(ClientException, exc_pattern):
swift_client.get_object(self.member_url, self.member_token, swift_client.get_object(self.member_url, self.member_token,
'myobjects', 'test_object', self.container, 'test_object',
headers=headers) headers=headers)
def test_storlet_acl_get_success(self): def test_storlet_acl_get_success(self):
@@ -133,24 +125,24 @@ class TestTestStorlet(StorletPythonFunctionalTest):
exc_pattern = '^.*403 Forbidden.*$' exc_pattern = '^.*403 Forbidden.*$'
with self.assertRaisesRegexp(ClientException, exc_pattern): with self.assertRaisesRegexp(ClientException, exc_pattern):
swift_client.get_object(self.member_url, self.member_token, swift_client.get_object(self.member_url, self.member_token,
'myobjects', 'test_object', self.container, 'test_object',
headers=headers) headers=headers)
headers = {'X-Storlet-Container-Read': self.conf.member_user, headers = {'X-Storlet-Container-Read': self.conf.member_user,
'X-Storlet-Name': self.storlet_name} 'X-Storlet-Name': self.storlet_name}
swift_client.post_container(self.url, swift_client.post_container(self.url,
self.token, self.token,
'myobjects', self.container,
headers) headers)
swift_client.head_container(self.url, swift_client.head_container(self.url,
self.token, self.token,
'myobjects') self.container)
headers = {'X-Run-Storlet': self.storlet_name} headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers) headers.update(self.additional_headers)
resp_dict = dict() resp_dict = dict()
swift_client.get_object(self.member_url, swift_client.get_object(self.member_url,
self.member_token, self.member_token,
'myobjects', 'test_object', self.container, 'test_object',
response_dict=resp_dict, response_dict=resp_dict,
headers=headers) headers=headers)
self.assertEqual(200, resp_dict['status']) self.assertEqual(200, resp_dict['status'])

7
tox.ini
View File

@@ -40,7 +40,12 @@ commands = {[testenv:py34]commands}
basepython = python2.7 basepython = python2.7
deps = deps =
-r{toxinidir}/test-requirements.txt -r{toxinidir}/test-requirements.txt
commands = ./.functests jenkins git+git://github.com/openstack/swift.git
setenv =
VIRTUAL_ENV={envdir}
STORLET_SAMPLE_PATH={toxinidir}/StorletSamples
CLUSTER_CONF_DIR={toxinidir}
commands = {toxinidir}/.functests jenkins
[testenv:venv] [testenv:venv]
commands = {posargs} commands = {posargs}