openstack/storlets
Code Issues Proposed changes

Introducing initial cluster config

Browse Source 
This is a baby step towards having a single cluster config that
serves:
(1) installation
(2) tests
(3) storlet mgmt
(4) various utils

At this point the functional tests and a deploy
storlet tool use a VERY initial cluster config

Change-Id: I80c4cb8c09adc2a8124f303e60489bdf1cba94b7
This commit is contained in:
Eran Rom
2016-01-04 19:36:26 +02:00
parent e78ef6d44d
commit 3773503e6e
16 changed files with 658 additions and 693 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@@ -22,8 +22,9 @@ lib64
# Installer logs # Installer logs
pip-log.txt pip-log.txt
# Deploy created # auto created for deploy
/Deploy/plabook/deploy Deploy/plabook/deploy/
cluster_config.json
# Unit test / coverage reports # Unit test / coverage reports
.coverage .coverage

11
cluster_config.json-sample Normal file
View File

@@ -0,0 +1,11 @@
{
"proxy_protocol": "HTTP",
"auth_protocol": "HTTP",
"auth_ip": "127.0.0.1",
"proxy_port": "80",
"auth_port": "5000",
"region": "",
"account": "service",
"user_name": "swift",
"password": "passw0rd"
}

0
common/__init__.py Normal file
View File

36
common/cluster_config_parser.py Normal file
View File

@@ -0,0 +1,36 @@
'''-------------------------------------------------------------------------
Copyright IBM Corp. 2015, 2016 All Rights Reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
Limitations under the License.
-------------------------------------------------------------------------'''
'''
*VERY* initial cluster configuration file
The intention is to have this as the single source
for all cluster information needs such as:
- Swift Install
- Storlets Install
- Tests
- Deploy storlets tools
- etc.
'''
import json
class ClusterConfig(object):
def __init__(self, config_path):
conf_string = open(config_path, 'r').read()
self.conf = json.loads(conf_string)
def get_conf(self):
return self.conf

63
common/deploy_storlet.py Normal file
View File

@@ -0,0 +1,63 @@
'''-------------------------------------------------------------------------
Copyright IBM Corp. 2015, 2016 All Rights Reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
Limitations under the License.
-------------------------------------------------------------------------'''
import os
import sys
import zipfile
from cluster_config_parser import ClusterConfig
from utils import storlet_get_auth, deploy_storlet
def list_classes(storlet_jar):
z = zipfile.ZipFile(storlet_jar, 'r')
for f in z.infolist():
name = f.filename
if name.endswith(".class"):
print('\t* ' + name[0:len(name) - 6].replace('/', '.'))
z.close()
def usage():
print("Useage: deploy_storlet.py <path to conf>")
def main():
if len(sys.argv) != 2:
usage()
sys.exit(-1)
conf = ClusterConfig(sys.argv[1]).get_conf()
url, token = storlet_get_auth(conf)
sys.stdout.write("Enter absolute path to storlet jar file: ")
storlet_jar = sys.stdin.readline().rstrip()
print("Your jar file contains the following classes:")
list_classes(storlet_jar)
sys.stdout.write("Please enter fully qualified storlet main class " +
"(choose from the list above): ")
storlet_main_class = sys.stdin.readline().rstrip()
print("Please enter dependency jars (leave a blank line when you are "
"done):")
dependency_jars = []
dependencies = []
dependency_jar = sys.stdin.readline().rstrip()
while dependency_jar:
dependency_jars.append(dependency_jar)
dependencies.append(os.path.basename(dependency_jar))
dependency_jar = sys.stdin.readline().rstrip()
deploy_storlet(url, token, storlet_jar, storlet_main_class,
dependency_jars, dependencies)
if __name__ == "__main__":
main()

88
common/utils.py Normal file
View File

@@ -0,0 +1,88 @@
'''-------------------------------------------------------------------------
Copyright IBM Corp. 2015, 2016 All Rights Reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
Limitations under the License.
-------------------------------------------------------------------------'''
import os
from swiftclient import client as c
def put_local_file(url, token, container,
local_path, local_file,
headers=None):
resp = dict()
f = open('%s/%s' % (local_path, local_file), 'r')
c.put_object(url, token, container, local_file, f,
headers=headers,
content_type="application/octet-stream",
response_dict=resp)
f.close()
def put_storlet_object(url, token, storlet_name, storlet_path,
dependency, main_class):
headers = {'X-Object-Meta-Storlet-Language': 'Java',
'X-Object-Meta-Storlet-Interface-Version': '1.0',
'X-Object-Meta-Storlet-Dependency': dependency,
'X-Object-Meta-Storlet-Object-Metadata': 'no',
'X-Object-Meta-Storlet-Main': main_class}
put_local_file(url, token, 'storlet', storlet_path, storlet_name, headers)
def put_storlet_executable_dependencies(url, token, deps):
resp = dict()
for d in deps:
headers = {'X-Object-Meta-Storlet-Dependency-Version': '1',
'X-Object-Meta-Storlet-Dependency-Permissions': '0755'}
f = open('%s' % d, 'r')
c.put_object(url, token, 'dependency', os.path.basename(d), f,
content_type="application/octet-stream",
headers=headers,
response_dict=resp)
f.close()
status = resp.get('status')
assert (status == 200 or status == 201)
def deploy_storlet(url, token, storlet_jar,
storlet_main_class,
dependencies):
# No need to create containers every time
# put_storlet_containers(url, token)
put_storlet_object(url, token,
os.path.basename(storlet_jar),
os.path.dirname(storlet_jar),
','.join(os.path.basename(x) for x in dependencies),
storlet_main_class)
put_storlet_executable_dependencies(url, token, dependencies)
def storlet_get_auth(conf):
auth_ip = conf['auth_ip']
auth_port = conf['auth_port']
account = conf['account']
user = conf['user_name']
passwd = conf['password']
os_options = {'tenant_name': account}
if conf['region']:
os_options['region_name'] = conf['region']
url, token = c.get_auth('http://' + auth_ip + ":" +
auth_port + '/v2.0',
account + ':' + user,
passwd,
os_options=os_options,
auth_version='2.0')
return url, token

1
test-requirements.txt
View File

@@ -7,6 +7,7 @@ hacking<0.11,>=0.10.0
coverage>=3.6 coverage>=3.6
discover discover
nose nose
pexpect
python-subunit>=0.0.18 python-subunit>=0.0.18
sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
oslotest>=1.2.0 # Apache-2.0 oslotest>=1.2.0 # Apache-2.0

74
tests/functional/__init__.py
View File

@@ -17,58 +17,30 @@ Limitations under the License.
@author: gilv / cdoron / evgenyl @author: gilv / cdoron / evgenyl
''' '''
from swiftclient import client as c import unittest
from common.cluster_config_parser import ClusterConfig
from common.utils import storlet_get_auth, deploy_storlet,\
put_local_file
PROXY_PROTOCOL = 'HTTP' CONFIG_FILE = '../../cluster_config.json'
AUTH_PROTOCOL = 'HTTP' PATH_TO_STORLETS = '../../StorletSamples'
DEV_AUTH_IP = '127.0.0.1' BIN_DIR = 'bin'
AUTH_IP = DEV_AUTH_IP
PROXY_PORT = '80'
AUTH_PORT = '5000'
ACCOUNT = 'service'
USER_NAME = 'swift'
PASSWORD = 'passw0rd'
def put_storlet_object(url, token, storlet_name, storlet_path, class StorletFunctionalTest(unittest.TestCase):
dependency, main_class): def setUp(self):
metadata = {'X-Object-Meta-Storlet-Language': 'Java', conf = ClusterConfig(CONFIG_FILE).get_conf()
'X-Object-Meta-Storlet-Interface-Version': '1.0', self.url, self.token = storlet_get_auth(conf)
'X-Object-Meta-Storlet-Dependency': dependency, path_to_bundle = '%s/%s/%s' % (PATH_TO_STORLETS, self.storlet_dir,
'X-Object-Meta-Storlet-Object-Metadata': 'no', BIN_DIR)
'X-Object-Meta-Storlet-Main': main_class} self.deps = []
f = open('%s/%s' % (storlet_path, storlet_name), 'r') for d in self.dep_names:
content_length = None self.deps.append('%s/%s' % (path_to_bundle, d))
response = dict() storlet = '%s/%s' % (path_to_bundle, self.storlet_name)
c.put_object(url, token, 'storlet', storlet_name, f,
content_length, None, None,
"application/octet-stream", metadata,
None, None, None, response)
f.close()
status = response.get('status')
assert (status == 200 or status == 201)
deploy_storlet(self.url, self.token,
def put_file_as_storlet_input_object(url, token, local_path, local_file): storlet, self.storlet_main,
resp = dict() self.deps)
f = open('%s/%s' % (local_path, local_file), 'r') if self.storlet_file:
c.put_object(url, token, 'myobjects', local_file, f, put_local_file(self.url, self.token, self.container,
content_type="application/octet-stream", path_to_bundle, self.storlet_file, self.headers)
response_dict=resp)
f.close()
status = resp.get('status')
assert (status == 200 or status == 201)
def put_dependency(url, token, local_path_to_dep, dep_name):
metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1'}
f = open('%s/%s' % (local_path_to_dep, dep_name), 'r')
content_length = None
response = dict()
c.put_object(url, token, 'dependency', dep_name, f,
content_length, None, None, "application/octet-stream",
metadata, None, None, None, response)
f.close()
status = response.get('status')
assert (status == 200 or status == 201)

237
tests/functional/test_SLO.py
View File

@@ -13,24 +13,12 @@ See the License for the specific language governing permissions and
Limitations under the License. Limitations under the License.
-------------------------------------------------------------------------''' -------------------------------------------------------------------------'''
from __init__ import ACCOUNT
from __init__ import AUTH_IP
from __init__ import AUTH_PORT
from __init__ import PASSWORD
from __init__ import put_storlet_object
from __init__ import USER_NAME
import json import json
import os import os
import random import random
import string import string
from swiftclient import client as c from swiftclient import client as c
from test_identity_storlet import deploy_identity_storlet from __init__ import StorletFunctionalTest
from test_identity_storlet import IDENTITY_STORLET_NAME
import unittest
SLOIDENTITY_PATH_TO_BUNDLE = '../../StorletSamples/SLOIdentityStorlet/bin'
SLOIDENTITY_STORLET_NAME = 'sloidentitystorlet-1.0.jar'
def create_local_chunks(): def create_local_chunks():
@@ -48,42 +36,40 @@ def delete_local_chunks():
os.remove(oname) os.remove(oname)
def put_SLO(url, token): def create_container(url, token, name):
# Create temp files
assembly = []
for i in range(1, 10):
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
content_length = None
response = dict() response = dict()
c.put_object(url, token, 'myobjects', oname, f, c.put_container(url, token, name, headers=None, response_dict=response)
content_length, None, None, "application/octet-stream",
None, None, None, None, response)
f.close()
status = response.get('status') status = response.get('status')
assert (status >= 200 and status < 300) assert (status >= 200 or status < 300)
headers = response.get('headers')
segment = dict()
segment['path'] = 'myobjects/%s' % oname
segment['size_bytes'] = 1048576
segment['etag'] = headers['etag']
assembly.append(segment)
content_length = None class TestSLO(StorletFunctionalTest):
def setUp(self):
self.storlet_dir = 'IdentityStorlet'
self.storlet_name = 'identitystorlet-1.0.jar'
self.storlet_main = 'com.ibm.storlet.identity.IdentityStorlet'
self.storlet_log = 'identitystorlet-1.0.log'
self.headers = None
self.storlet_file = ''
self.container = 'myobjects'
self.dep_names = []
super(TestSLO, self).setUp()
create_container(self.url, self.token, 'myobjects')
create_container(self.url, self.token, 'container1')
create_container(self.url, self.token, 'container2')
create_container(self.url, self.token, 'container3')
create_local_chunks()
self.put_SLO()
self.get_SLO()
def tearDown(self):
delete_local_chunks()
def get_SLO(self):
response = dict() response = dict()
headers = {'x-object-meta-prop1': 'val1'} headers, body = c.get_object(self.url, self.token,
c.put_object(url, token, 'myobjects', 'assembly', json.dumps(assembly), self.container, 'assembly',
content_length=None, etag=None, chunk_size=None,
headers=headers, query_string='multipart-manifest=put',
response_dict=response)
status = response.get('status')
assert (status >= 200 and status < 300)
def get_SLO(url, token):
response = dict()
headers, body = c.get_object(url, token, 'myobjects', 'assembly',
http_conn=None, resp_chunk_size=1048576, http_conn=None, resp_chunk_size=1048576,
query_string=None, response_dict=response, query_string=None, response_dict=response,
headers=None) headers=None)
@@ -95,12 +81,46 @@ def get_SLO(url, token):
file_content = f.read() file_content = f.read()
# print '%s %s' % (chunk[:10], file_content[:10]) # print '%s %s' % (chunk[:10], file_content[:10])
# print '%d %d' % (len(chunk), len(file_content)) # print '%d %d' % (len(chunk), len(file_content))
assert(chunk == file_content) self.assertEqual(chunk, file_content)
f.close() f.close()
i = i + 1 i = i + 1
def put_SLO(self):
# Create temp files
assembly = []
for i in range(1, 10):
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
content_length = None
response = dict()
c.put_object(self.url, self.token,
self.container, oname, f,
content_length, None, None,
"application/octet-stream",
None, None, None, None, response)
f.close()
status = response.get('status')
self.assertTrue(status >= 200 and status < 300)
def compare_slo_to_chunks(body): headers = response.get('headers')
segment = dict()
segment['path'] = 'myobjects/%s' % oname
segment['size_bytes'] = 1048576
segment['etag'] = headers['etag']
assembly.append(segment)
content_length = None
response = dict()
headers = {'x-object-meta-prop1': 'val1'}
c.put_object(self.url, self.token, self.container,
'assembly', json.dumps(assembly),
content_length=None, etag=None, chunk_size=None,
headers=headers, query_string='multipart-manifest=put',
response_dict=response)
status = response.get('status')
self.assertTrue(status >= 200 and status < 300)
def compare_slo_to_chunks(self, body):
i = 1 i = 1
for chunk in body: for chunk in body:
if chunk: if chunk:
@@ -110,7 +130,7 @@ def compare_slo_to_chunks(body):
file_content = f.read() file_content = f.read()
# print '%s %s' % (chunk[:10], file_content[:10]) # print '%s %s' % (chunk[:10], file_content[:10])
# print '%d %d' % (len(chunk), len(file_content)) # print '%d %d' % (len(chunk), len(file_content))
assert(chunk == file_content) self.assertEqual(chunk, file_content)
f.close() f.close()
i = i + 1 i = i + 1
else: else:
@@ -120,126 +140,15 @@ def compare_slo_to_chunks(body):
f = open(oname, 'r') f = open(oname, 'r')
aux_content += f.read() aux_content += f.read()
f.close() f.close()
assert(chunk == aux_content) self.ssertEqual(chunk, aux_content)
def test_get_SLO(self):
def invoke_identity_on_get_SLO(url, token): headers = {'X-Run-Storlet': self.storlet_name}
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
response = dict() response = dict()
headers, body = c.get_object(url, token, headers, body = c.get_object(self.url, self.token,
'myobjects', 'assembly', 'myobjects', 'assembly',
query_string=None, query_string=None,
response_dict=response, response_dict=response,
resp_chunk_size=1048576, resp_chunk_size=1048576,
headers=metadata) headers=headers)
compare_slo_to_chunks(body) self.compare_slo_to_chunks(body)
def invoke_identity_on_get_SLO_double(url, token):
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
response = dict()
headers, body = c.get_object(url, token,
'myobjects',
'assembly',
query_string='double=true',
response_dict=response,
resp_chunk_size=2048,
headers=metadata)
i = 1
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
file_content = f.read()
j = 0 # Count chunks in file 1...1024
for chunk in body:
file_fragment = file_content[j * 1024:(j + 1) * 1024]
chunk_framgment_low = chunk[0:1024]
chunk_framgment_high = chunk[1024:2048]
assert(chunk_framgment_low == file_fragment)
assert(chunk_framgment_high == file_fragment)
j = j + 1
if j == 1024:
i = i + 1
if i == 10:
break
f.close()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
file_content = f.read()
j = 0
assert i == 10
def invoke_identity_on_partial_get_SLO(url, token):
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
for i in range(5):
response = dict()
headers, body = c.get_object(url, token,
'myobjects',
'assembly',
query_string=None,
response_dict=response,
resp_chunk_size=1048576,
headers=metadata)
j = 1
for chunk in body:
j = j + 1
if j == 5:
break
# def delete_files():
# for i in range(1,4):
# fname = '/tmp/aux_file%d' % i
# os.remove(fname)
def create_container(url, token, name):
response = dict()
c.put_container(url, token, name, headers=None, response_dict=response)
status = response.get('status')
assert (status >= 200 or status < 300)
def deploy_sloidentity_storlet(url, token):
response = dict()
c.put_container(url, token, 'mysloobject', None, None, response)
status = response.get('status')
assert (status >= 200 or status < 300)
put_storlet_object(url, token,
SLOIDENTITY_STORLET_NAME,
SLOIDENTITY_PATH_TO_BUNDLE,
'',
'com.ibm.storlet.sloidentity.SLOIdentityStorlet')
class TestSLO(unittest.TestCase):
def setUp(self):
os_options = {'tenant_name': ACCOUNT}
self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+ "/v2.0", ACCOUNT + ":" + USER_NAME,
PASSWORD, os_options=os_options,
auth_version="2.0")
create_container(self.url, self.token, 'myobjects')
create_container(self.url, self.token, 'container1')
create_container(self.url, self.token, 'container2')
create_container(self.url, self.token, 'container3')
create_local_chunks()
put_SLO(self.url, self.token)
get_SLO(self.url, self.token)
deploy_identity_storlet(self.url, self.token)
def tearDown(self):
delete_local_chunks()
def test_get_SLO(self):
invoke_identity_on_get_SLO(self.url, self.token)
# YM comment out 2 lines - temporary only!
# progress_msg("Invoking storlet on SLO in GET with double")
# invoke_identity_on_get_SLO_double(url, token)
# progress_msg("Invoking storlet on SLO in partial GET")
# invoke_identity_on_partial_get_SLO(url, token)

49
tests/functional/test_deploy_storlet.py Normal file
View File

@@ -0,0 +1,49 @@
'''-------------------------------------------------------------------------
Copyright IBM Corp. 2015, 2015 All Rights Reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
Limitations under the License.
-------------------------------------------------------------------------'''
import unittest
import pexpect
from __init__ import PATH_TO_STORLETS, BIN_DIR
DEPLOY_STORLET_PATH = '../../common/deploy_storlet.py'
CONF_PATH = '../../cluster_config.json'
EXECDEP_STORLET_PATH = '%s/%s/%s' % (PATH_TO_STORLETS,
'ExecDepStorlet',
BIN_DIR)
EXECDEP_STORLET_PATH = '%s/%s' % (EXECDEP_STORLET_PATH,
'execdepstorlet-1.0.jar')
EXECDEP_STORLET_DEP_PATH = '%s/%s' % (EXECDEP_STORLET_PATH,
'get42')
class TestExecDepStorlet(unittest.TestCase):
def test_deploy_storlet_util(self):
child = pexpect.spawn('python %s %s' % (DEPLOY_STORLET_PATH,
CONF_PATH))
try:
child.expect('Enter absolute path to storlet jar file.*:',
timeout=1)
child.sendline(EXECDEP_STORLET_PATH)
child.expect('com.ibm.storlet.execdep.ExecDepStorlet',
timeout=1)
child.expect('Please enter fully qualified storlet main class.*',
timeout=1)
child.sendline('com.ibm.storlet.execdep.ExecDepStorlet')
child.expect('Please enter dependency.*', timeout=1)
child.sendline(EXECDEP_STORLET_DEP_PATH)
child.expect('\n')
except Exception as err:
self.fail('Unexpected exception %s' % err)

78
tests/functional/test_execdep_storlet.py
View File

@@ -13,73 +13,31 @@ See the License for the specific language governing permissions and
Limitations under the License. Limitations under the License.
-------------------------------------------------------------------------''' -------------------------------------------------------------------------'''
from __init__ import ACCOUNT
from __init__ import AUTH_IP
from __init__ import AUTH_PORT
from __init__ import PASSWORD
from __init__ import put_file_as_storlet_input_object
from __init__ import put_storlet_object
from __init__ import USER_NAME
from swiftclient import client as c from swiftclient import client as c
import unittest from __init__ import StorletFunctionalTest
EXECDEP_PATH_TO_BUNDLE = '../../StorletSamples/ExecDepStorlet/bin/'
EXECDEP_STORLET_NAME = 'execdepstorlet-1.0.jar'
EXECDEP_STORLET_LOG_NAME = 'execdepstorlet-1.0.log'
EXECDEP_JUNK_FILE = 'junk.txt'
EXECDEP_DEPS_NAMES = ['get42']
class TestExexDepStorlet(unittest.TestCase): class TestExecDepStorlet(StorletFunctionalTest):
def setUp(self): def setUp(self):
os_options = {'tenant_name': ACCOUNT} self.storlet_dir = 'ExecDepStorlet'
self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT self.storlet_name = 'execdepstorlet-1.0.jar'
+ "/v2.0", ACCOUNT + ":" + USER_NAME, self.storlet_main = 'com.ibm.storlet.execdep.ExecDepStorlet'
PASSWORD, os_options=os_options, self.storlet_log = 'execdepstorlet-1.0.log'
auth_version="2.0") self.headers = None
self.storlet_file = 'junk.txt'
self.container = 'myobjects'
self.dep_names = ['get42']
super(TestExecDepStorlet, self).setUp()
def put_storlet_executable_dependencies(self): def test_execdep(self):
resp = dict() headers = {'X-Run-Storlet': self.storlet_name}
for d in EXECDEP_DEPS_NAMES:
metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1',
'X-Object-Meta-Storlet-Dependency-Permissions': '0755'}
f = open('%s/%s' % (EXECDEP_PATH_TO_BUNDLE, d), 'r')
c.put_object(self.url, self.token, 'dependency', d, f,
content_type="application/octet-stream",
headers=metadata,
response_dict=resp)
f.close()
status = resp.get('status')
assert (status == 200 or status == 201)
def deploy_storlet(self):
# No need to create containers every time
# put_storlet_containers(url, token)
put_storlet_object(self.url, self.token,
EXECDEP_STORLET_NAME,
EXECDEP_PATH_TO_BUNDLE,
','.join(str(x) for x in EXECDEP_DEPS_NAMES),
'com.ibm.storlet.execdep.ExecDepStorlet')
self.put_storlet_executable_dependencies()
put_file_as_storlet_input_object(self.url,
self.token,
EXECDEP_PATH_TO_BUNDLE,
EXECDEP_JUNK_FILE)
def invoke_storlet(self):
metadata = {'X-Run-Storlet': EXECDEP_STORLET_NAME}
resp = dict() resp = dict()
resp_headers, gf = c.get_object(self.url, self.token, resp_headers, gf = c.get_object(self.url, self.token,
'myobjects', 'myobjects',
EXECDEP_JUNK_FILE, self.storlet_file,
response_dict=resp, response_dict=resp,
headers=metadata) headers=headers)
assert 'x-object-meta-depend-ret-code' in resp_headers self.assertTrue('x-object-meta-depend-ret-code' in resp_headers)
assert resp_headers['x-object-meta-depend-ret-code'] == '42' self.assertTrue(resp_headers['x-object-meta-depend-ret-code'] == '42')
assert resp['status'] == 200 self.assertEqual(resp['status'], 200)
def test_execdep(self):
self.deploy_storlet()
self.invoke_storlet()

117
tests/functional/test_half_storlet.py
View File

@@ -13,49 +13,25 @@ See the License for the specific language governing permissions and
Limitations under the License. Limitations under the License.
-------------------------------------------------------------------------''' -------------------------------------------------------------------------'''
from __init__ import ACCOUNT
from __init__ import AUTH_IP
from __init__ import AUTH_PORT
from __init__ import PASSWORD
from __init__ import put_storlet_object
from __init__ import USER_NAME
import random import random
import string import string
from swiftclient import client as c from swiftclient import client as c
import unittest from __init__ import StorletFunctionalTest
# Test Constants class TestHalfIdentityStorlet(StorletFunctionalTest):
HALF_PATH_TO_BUNDLE = '../../StorletSamples/HalfStorlet/bin/' def setUp(self):
HALF_STORLET_NAME = 'halfstorlet-1.0.jar' self.storlet_dir = 'HalfStorlet'
HALF_SOURCE_FILE = 'source.txt' self.storlet_name = 'halfstorlet-1.0.jar'
self.storlet_main = 'com.ibm.storlet.half.HalfStorlet'
self.storlet_log = ''
self.headers = {'X-Object-Meta-Testkey': 'tester'}
self.storlet_file = 'source.txt'
self.container = 'myobjects'
self.dep_names = []
super(TestHalfIdentityStorlet, self).setUp()
def invoke_storlet(self, op, params=None, global_params=None,
def put_storlet_input_object(url, token):
resp = dict()
metadata = {'X-Object-Meta-Testkey': 'tester'}
f = open('%s/%s' % (HALF_PATH_TO_BUNDLE, HALF_SOURCE_FILE), 'r')
c.put_object(url, token, 'myobjects', HALF_SOURCE_FILE, f,
content_type="application/octet-stream",
headers=metadata,
response_dict=resp)
f.close()
status = resp.get('status')
assert (status == 200 or status == 201)
def deploy_storlet(url, token):
# No need to create containers every time
# put_storlet_containers(url, token)
put_storlet_object(url, token,
HALF_STORLET_NAME,
HALF_PATH_TO_BUNDLE,
'',
'com.ibm.storlet.half.HalfStorlet')
put_storlet_input_object(url, token)
def invoke_storlet(url, token, op, params=None, global_params=None,
headers=None): headers=None):
if params is not None: if params is not None:
querystring = '' querystring = ''
@@ -65,29 +41,31 @@ def invoke_storlet(url, token, op, params=None, global_params=None,
else: else:
querystring = None querystring = None
metadata = {'X-Run-Storlet': HALF_STORLET_NAME} req_headers = {'X-Run-Storlet': self.storlet_name}
if headers: if headers:
metadata.update(headers) req_headers.update(headers)
if op == 'GET': if op == 'GET':
# Get original object # Get original object
original_headers, original_content = \ original_h, original_c = \
c.get_object(url, token, 'myobjects', HALF_SOURCE_FILE, c.get_object(self.url, self.token, 'myobjects',
self.storlet_file,
response_dict=dict()) response_dict=dict())
# print original_headers # print original_headers
file_length = int(original_headers['content-length']) file_length = int(original_h['content-length'])
processed_headers, returned_content = \ processed_h, returned_c = \
c.get_object(url, token, 'myobjects', HALF_SOURCE_FILE, c.get_object(self.url, self.token, 'myobjects',
self.storlet_file,
query_string=querystring, response_dict=dict(), query_string=querystring, response_dict=dict(),
headers=metadata, resp_chunk_size=file_length) headers=req_headers, resp_chunk_size=file_length)
processed_content = '' processed_c = ''
for chunk in returned_content: for chunk in returned_c:
if chunk: if chunk:
processed_content += chunk processed_c += chunk
assert(original_headers['X-Object-Meta-Testkey'.lower()] == self.assertEqual(original_h['X-Object-Meta-Testkey'.lower()],
processed_headers['X-Object-Meta-Testkey'.lower()]) processed_h['X-Object-Meta-Testkey'.lower()])
return processed_content return processed_c
if op == 'PUT': if op == 'PUT':
# PUT a random file # PUT a random file
@@ -98,41 +76,36 @@ def invoke_storlet(url, token, op, params=None, global_params=None,
string.digits) for _ in range(32)) string.digits) for _ in range(32))
# content_length = 1024 # content_length = 1024
content_length = None content_length = None
headers = {'X-Run-Storlet': HALF_STORLET_NAME, headers = {'X-Run-Storlet': self.storlet_name,
'X-Object-Meta-Testkey': random_md} 'X-Object-Meta-Testkey': random_md}
c.put_object(url, token, 'myobjects', 'half_random_source', c.put_object(self.url, self.token, self.container,
'half_random_source',
uploaded_content, content_length, None, None, uploaded_content, content_length, None, None,
"application/octet-stream", headers, None, None, "application/octet-stream", headers, None, None,
querystring, response) querystring, response)
resp_headers, saved_content = c.get_object(url, token, 'myobjects', resp_headers, saved_content = c.get_object(self.url, self.token,
'myobjects',
'half_random_source', 'half_random_source',
response_dict=dict()) response_dict=dict())
if params is not None and params.get('double', None) == 'true': if params is not None and params.get('double', None) == 'true':
assert(uploaded_content == saved_content[:1024]) self.assertEqual(uploaded_content, saved_content[:1024])
assert(uploaded_content == saved_content[1024:]) self.assertEqual(uploaded_content, saved_content[1024:])
else: else:
assert(uploaded_content == saved_content) self.assertEqual(uploaded_content, saved_content)
if params is not None and params.get('execute', None) is not None: if params is not None and params.get('execute', None) is not None:
assert(resp_headers['X-Object-Meta-Execution result'.lower()] == self.assertEqual(
resp_headers['X-Object-Meta-Execution result'.lower()],
'42') '42')
assert(resp_headers['X-Object-Meta-Testkey'.lower()] == random_md) self.assertEqual(resp_headers['X-Object-Meta-Testkey'.lower()],
random_md)
class TestHalfIdentityStorlet(unittest.TestCase):
def setUp(self):
os_options = {'tenant_name': ACCOUNT}
self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+ "/v2.0", ACCOUNT + ":" + USER_NAME,
PASSWORD, os_options=os_options,
auth_version='2.0')
deploy_storlet(self.url, self.token)
def test_get(self): def test_get(self):
assert (invoke_storlet(self.url, self.token, 'GET') == 'acegikmn') res = self.invoke_storlet('GET')
self.assertEqual(res, 'acegikmn')
def test_get_range(self): def test_get_range(self):
assert (invoke_storlet(self.url, self.token, 'GET', res = self.invoke_storlet('GET', headers={'range': 'bytes=5-10'})
headers={'range': 'bytes=5-10'}) == 'fhj') self.assertEqual(res, 'fhj')

188
tests/functional/test_identity_storlet.py
View File

@@ -13,87 +13,26 @@ See the License for the specific language governing permissions and
Limitations under the License. Limitations under the License.
-------------------------------------------------------------------------''' -------------------------------------------------------------------------'''
from __init__ import ACCOUNT
from __init__ import AUTH_IP
from __init__ import AUTH_PORT
from __init__ import PASSWORD
from __init__ import put_storlet_object
from __init__ import USER_NAME
import os import os
import random import random
import string import string
from swiftclient import client as c from swiftclient import client as c
import unittest from __init__ import StorletFunctionalTest
# Test Constants class TestIdentityStorlet(StorletFunctionalTest):
IDENTITY_PATH_TO_BUNDLE = '../../StorletSamples/IdentityStorlet/bin/' def setUp(self):
IDENTITY_STORLET_NAME = 'identitystorlet-1.0.jar' self.storlet_dir = 'IdentityStorlet'
IDENTITY_STORLET_LOG_NAME = 'identitystorlet-1.0.log' self.storlet_name = 'identitystorlet-1.0.jar'
IDENTITY_SOURCE_FILE = 'source.txt' self.storlet_main = 'com.ibm.storlet.identity.IdentityStorlet'
IDENTITY_DEPS_NAMES = ['get42'] self.storlet_log = 'identitystorlet-1.0.log'
self.headers = {'X-Object-Meta-Testkey': 'tester'}
self.storlet_file = 'source.txt'
self.container = 'myobjects'
self.dep_names = ['get42']
super(TestIdentityStorlet, self).setUp()
def invoke_storlet(self, op, params=None, global_params=None):
def put_storlet_executable_dependencies(url, token):
resp = dict()
for d in IDENTITY_DEPS_NAMES:
metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1',
'X-Object-Meta-Storlet-Dependency-Permissions': '0755'}
f = open('%s/%s' % (IDENTITY_PATH_TO_BUNDLE, d), 'r')
c.put_object(url, token, 'dependency', d, f,
content_type="application/octet-stream",
headers=metadata,
response_dict=resp)
f.close()
status = resp.get('status')
assert (status == 200 or status == 201)
def put_storlet_input_object(url, token):
resp = dict()
metadata = {'X-Object-Meta-Testkey': 'tester'}
f = open('%s/%s' % (IDENTITY_PATH_TO_BUNDLE, IDENTITY_SOURCE_FILE), 'r')
c.put_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE, f,
content_type="application/octet-stream",
headers=metadata,
response_dict=resp)
f.close()
status = resp.get('status')
assert (status == 200 or status == 201)
def deploy_identity_storlet(url, token):
# No need to create containers every time
# put_storlet_containers(url, token)
put_storlet_object(url, token,
IDENTITY_STORLET_NAME,
IDENTITY_PATH_TO_BUNDLE,
','.join(str(x) for x in IDENTITY_DEPS_NAMES),
'com.ibm.storlet.identity.IdentityStorlet')
put_storlet_executable_dependencies(url, token)
put_storlet_input_object(url, token)
def invoke_storlet_on_1GB_file(url, token):
GBFile = open('/tmp/1GB_file', 'w')
for _ in range(128):
uploaded_content = ''.join('1' for _ in range(8 * 1024 * 1024))
GBFile.write(uploaded_content)
GBFile.close()
headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
GBFile = open('/tmp/1GB_file', 'r')
response = dict()
c.put_object(url, token, 'myobjects', '1GBFile', GBFile,
1024 * 1024 * 1024, None, None, "application/octet-stream",
headers, None, None, None, response)
status = response.get('status')
assert (status == 200 or status == 201)
os.remove('/tmp/1GB_file')
def invoke_storlet(url, token, op, params=None, global_params=None):
if params is not None: if params is not None:
querystring = '' querystring = ''
for key in params: for key in params:
@@ -102,93 +41,102 @@ def invoke_storlet(url, token, op, params=None, global_params=None):
else: else:
querystring = None querystring = None
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME} headers = {'X-Run-Storlet': self.storlet_name}
if op == 'GET': if op == 'GET':
# Get original object # Get original object
original_headers, original_content = \ original_h, original_c = c.get_object(self.url, self.token,
c.get_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE, 'myobjects',
self.storlet_file,
response_dict=dict()) response_dict=dict())
# print original_headers # print original_headers
file_length = int(original_headers['content-length']) file_length = int(original_h['content-length'])
processed_headers, returned_content = \ processed_h, returned_c = c.get_object(
c.get_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE, self.url, self.token,
self.container, self.storlet_file,
query_string=querystring, response_dict=dict(), query_string=querystring, response_dict=dict(),
headers=metadata, resp_chunk_size=file_length) headers=headers, resp_chunk_size=file_length)
processed_content = '' processed_c = ''
for chunk in returned_content: for chunk in returned_c:
if chunk: if chunk:
processed_content += chunk processed_c += chunk
if params is not None and params.get('execute', None) is not None: if params is not None and params.get('execute', None) is not None:
assert(processed_headers['X-Object-Meta-Execution result'.lower()] mdv = processed_h['X-Object-Meta-Execution result'.lower()]
== '42') self.assertEqual(mdv, '42')
if params is not None and params.get('double', None) == 'true': if params is not None and params.get('double', None) == 'true':
assert(original_content == processed_content[:file_length]) self.assertEqual(original_c, processed_c[:file_length])
assert(original_content == processed_content[file_length:]) self.assertEqual(original_c, processed_c[file_length:])
else: else:
assert(original_content == processed_content) self.assertEqual(original_c, processed_c)
assert(original_headers['X-Object-Meta-Testkey'.lower()] == self.assertEqual(original_h['X-Object-Meta-Testkey'.lower()],
processed_headers['X-Object-Meta-Testkey'.lower()]) processed_h['X-Object-Meta-Testkey'.lower()])
if op == 'PUT': if op == 'PUT':
# PUT a random file # PUT a random file
response = dict() response = dict()
uploaded_content = ''.join(random.choice(string.ascii_uppercase + uploaded_c = ''.join(random.choice(string.ascii_uppercase +
string.digits) for _ in range(1024)) string.digits) for _ in range(1024))
random_md = ''.join(random.choice(string.ascii_uppercase + random_md = ''.join(random.choice(string.ascii_uppercase +
string.digits) for _ in range(32)) string.digits) for _ in range(32))
content_length = None content_length = None
headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME, headers = {'X-Run-Storlet': self.storlet_name,
'X-Object-Meta-Testkey': random_md} 'X-Object-Meta-Testkey': random_md}
c.put_object(url, token, 'myobjects', 'identity_random_source', c.put_object(self.url, self.token, self.container,
uploaded_content, content_length, None, None, 'identity_random_source',
uploaded_c, content_length, None, None,
"application/octet-stream", headers, None, None, "application/octet-stream", headers, None, None,
querystring, response) querystring, response)
resp_headers, saved_content = c.get_object(url, token, resp_headers, saved_c = c.get_object(self.url, self.token,
'myobjects', 'myobjects',
'identity_random_source', 'identity_random_source',
response_dict=dict()) response_dict=dict())
if params is not None and params.get('double', None) == 'true': if params is not None and params.get('double', None) == 'true':
assert(uploaded_content == saved_content[:1024]) assert(uploaded_c == saved_c[:1024])
assert(uploaded_content == saved_content[1024:]) assert(uploaded_c == saved_c[1024:])
else: else:
assert(uploaded_content == saved_content) assert(uploaded_c == saved_c)
if params is not None and params.get('execute', None) is not None: if params is not None and params.get('execute', None) is not None:
assert(resp_headers['X-Object-Meta-Execution result'.lower()] == mdv = resp_headers['X-Object-Meta-Execution result'.lower()]
'42') self.assertEqual(mdv, '42')
assert(resp_headers['X-Object-Meta-Testkey'.lower()] == random_md) self.assertEqual(resp_headers['X-Object-Meta-Testkey'.lower()],
random_md)
def test_put_1GB_file(self):
GBFile = open('/tmp/1GB_file', 'w')
for _ in range(128):
uploaded_content = ''.join('1' for _ in range(8 * 1024 * 1024))
GBFile.write(uploaded_content)
GBFile.close()
class TestIdentityStorlet(unittest.TestCase): headers = {'X-Run-Storlet': self.storlet_name}
def setUp(self): GBFile = open('/tmp/1GB_file', 'r')
os_options = {'tenant_name': ACCOUNT} response = dict()
self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT c.put_object(self.url, self.token,
+ "/v2.0", ACCOUNT + ":" + USER_NAME, self.container, '1GBFile', GBFile,
PASSWORD, os_options=os_options, 1024 * 1024 * 1024, None, None,
auth_version="2.0") "application/octet-stream",
headers, None, None, None, response)
deploy_identity_storlet(self.url, self.token) status = response.get('status')
self.assertTrue(status in [200, 201])
os.remove('/tmp/1GB_file')
def test_put(self): def test_put(self):
invoke_storlet(self.url, self.token, 'PUT') self.invoke_storlet('PUT')
def test_put_1GB(self):
invoke_storlet_on_1GB_file(self.url, self.token)
def test_put_execute(self): def test_put_execute(self):
invoke_storlet(self.url, self.token, 'PUT', {'execute': 'true'}) self.invoke_storlet('PUT', {'execute': 'true'})
def test_put_double(self): def test_put_double(self):
invoke_storlet(self.url, self.token, 'PUT', {'double': 'true'}) self.invoke_storlet('PUT', {'double': 'true'})
def test_get(self): def test_get(self):
invoke_storlet(self.url, self.token, 'GET') self.invoke_storlet('GET')
def test_get_double(self): def test_get_double(self):
invoke_storlet(self.url, self.token, 'GET', {'double': 'true'}) self.invoke_storlet('GET', {'double': 'true'})
def test_get_execute(self): def test_get_execute(self):
invoke_storlet(self.url, self.token, 'GET', {'execute': 'true'}) self.invoke_storlet('GET', {'execute': 'true'})

97
tests/functional/test_metadata_storlet.py
View File

@@ -13,25 +13,18 @@ See the License for the specific language governing permissions and
Limitations under the License. Limitations under the License.
----------------------------------------------------------------''' ----------------------------------------------------------------'''
from __init__ import ACCOUNT
from __init__ import AUTH_IP
from __init__ import AUTH_PORT
from __init__ import PASSWORD
from __init__ import put_storlet_object
from __init__ import USER_NAME
from swiftclient import client as c from swiftclient import client as c
import unittest from __init__ import StorletFunctionalTest
# Test Constants
PATH_TO_BUNDLE = '../../StorletSamples/TestMetadataStorlet/bin/'
STORLET_NAME = 'testmetadatastorlet-1.0.jar'
STORLET_LOG_NAME = 'testmetadatastorlet-1.0.log'
SOURCE_FILE = 'source.txt'
def put_storlet_input_object(url, token): class TestMetadataStorlet(StorletFunctionalTest):
resp = dict() def setUp(self):
metadata = {'X-Object-Meta-key1': '1', self.storlet_dir = 'TestMetadataStorlet'
self.storlet_name = 'testmetadatastorlet-1.0.jar'
self.storlet_main = ('com.ibm.storlet.testmetadatastorlet'
'.MetadataStorlet')
self.storlet_log = 'testmetadatastorlet-1.0.log'
self.headers = {'X-Object-Meta-key1': '1',
'X-Object-Meta-key2': '2', 'X-Object-Meta-key2': '2',
'X-Object-Meta-key3': '3', 'X-Object-Meta-key3': '3',
'X-Object-Meta-key4': '4', 'X-Object-Meta-key4': '4',
@@ -41,28 +34,12 @@ def put_storlet_input_object(url, token):
'X-Object-Meta-key8': '8', 'X-Object-Meta-key8': '8',
'X-Object-Meta-key9': '9', 'X-Object-Meta-key9': '9',
'X-Object-Meta-key10': '10'} 'X-Object-Meta-key10': '10'}
f = open('%s/%s' % (PATH_TO_BUNDLE, SOURCE_FILE), 'r') self.storlet_file = 'source.txt'
c.put_object(url, token, 'myobjects', SOURCE_FILE, f, self.container = 'myobjects'
content_type="application/octet-stream", self.dep_names = []
headers=metadata, super(TestMetadataStorlet, self).setUp()
response_dict=resp)
f.close()
status = resp.get('status')
assert (status == 200 or status == 201)
def test_metadata_get(self, params=None, global_params=None):
def deploy_storlet(url, token):
# No need to create containers every time
# put_storlet_containers(url, token)
put_storlet_object(url, token,
STORLET_NAME,
PATH_TO_BUNDLE,
'',
'com.ibm.storlet.testmetadatastorlet.MetadataStorlet')
put_storlet_input_object(url, token)
def invoke_storlet(url, token, op, params=None, global_params=None):
if params is not None: if params is not None:
querystring = '' querystring = ''
for key in params: for key in params:
@@ -71,34 +48,20 @@ def invoke_storlet(url, token, op, params=None, global_params=None):
else: else:
querystring = None querystring = None
metadata = {'X-Run-Storlet': STORLET_NAME} headers = {'X-Run-Storlet': self.storlet_name}
if op == 'GET':
# Get original object
original_headers, original_content = \ original_headers, original_content = \
c.get_object(url, token, 'myobjects', SOURCE_FILE, c.get_object(self.url, self.token,
response_dict=dict(), headers=metadata) 'myobjects', self.storlet_file,
assert(original_headers['X-Object-Meta-key1'.lower()] == '1') response_dict=dict(), headers=headers)
assert(original_headers['X-Object-Meta-key2'.lower()] == '2') self.assertEqual(original_headers['X-Object-Meta-key1'.lower()], '1')
assert(original_headers['X-Object-Meta-key3'.lower()] == '3') self.assertEqual(original_headers['X-Object-Meta-key2'.lower()], '2')
assert(original_headers['X-Object-Meta-key4'.lower()] == '4') self.assertEqual(original_headers['X-Object-Meta-key3'.lower()], '3')
assert(original_headers['X-Object-Meta-key5'.lower()] == '5') self.assertEqual(original_headers['X-Object-Meta-key4'.lower()], '4')
assert(original_headers['X-Object-Meta-key6'.lower()] == '6') self.assertEqual(original_headers['X-Object-Meta-key5'.lower()], '5')
assert(original_headers['X-Object-Meta-key7'.lower()] == '7') self.assertEqual(original_headers['X-Object-Meta-key6'.lower()], '6')
assert(original_headers['X-Object-Meta-key8'.lower()] == '8') self.assertEqual(original_headers['X-Object-Meta-key7'.lower()], '7')
assert(original_headers['X-Object-Meta-key9'.lower()] == '9') self.assertEqual(original_headers['X-Object-Meta-key8'.lower()], '8')
assert(original_headers['X-Object-Meta-key10'.lower()] == '10') self.assertEqual(original_headers['X-Object-Meta-key9'.lower()], '9')
assert(original_headers['X-Object-Meta-override_key'.lower()] == self.assertEqual(original_headers['X-Object-Meta-key10'.lower()], '10')
'new_value') omv = original_headers['X-Object-Meta-override_key'.lower()]
self.assertEqual(omv, 'new_value')
class TestMetadataStorlet(unittest.TestCase):
def setUp(self):
os_options = {'tenant_name': ACCOUNT}
self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+ "/v2.0", ACCOUNT + ":" + USER_NAME,
PASSWORD, os_options=os_options,
auth_version="2.0")
deploy_storlet(self.url, self.token)
def test_metadata_get(self):
invoke_storlet(self.url, self.token, 'GET')

57
tests/functional/test_test_storlet.py
View File

@@ -13,20 +13,9 @@ See the License for the specific language governing permissions and
Limitations under the License. Limitations under the License.
-------------------------------------------------------------------------''' -------------------------------------------------------------------------'''
from __init__ import ACCOUNT
from __init__ import AUTH_IP
from __init__ import AUTH_PORT
from __init__ import PASSWORD
from __init__ import put_storlet_object
from __init__ import USER_NAME
import threading import threading
import unittest
from swiftclient import client as c from swiftclient import client as c
from __init__ import StorletFunctionalTest
TEST_STORLET_NAME = 'test-10.jar'
PATH_TO_STORLETS = '../../StorletSamples'
class myTestThread (threading.Thread): class myTestThread (threading.Thread):
@@ -40,27 +29,26 @@ class myTestThread (threading.Thread):
self.test_class.invokeTestStorlet("print", False) self.test_class.invokeTestStorlet("print", False)
class TestTestStorlet(unittest.TestCase): class TestTestStorlet(StorletFunctionalTest):
def setUp(self): def setUp(self):
os_options = {'tenant_name': ACCOUNT} self.storlet_dir = 'TestStorlet'
self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT self.storlet_name = 'test-10.jar'
+ "/v2.0", ACCOUNT + ":" + USER_NAME, self.storlet_main = 'com.ibm.storlet.test.test1'
PASSWORD, os_options=os_options, self.storlet_log = ''
auth_version='2.0') self.headers = None
put_storlet_object(self.url, self.storlet_file = ''
self.token, self.container = 'myobjects'
TEST_STORLET_NAME, self.dep_names = []
"%s/TestStorlet/bin" % PATH_TO_STORLETS, super(TestTestStorlet, self).setUp()
'',
'com.ibm.storlet.test.test1')
c.put_object(self.url, c.put_object(self.url,
self.token, self.token,
'myobjects', self.container,
'test_object', 'test_object',
'some content') 'some content')
def invokeTestStorlet(self, op, withlog=False): def invokeTestStorlet(self, op, withlog=False):
headers = {'X-Run-Storlet': TEST_STORLET_NAME} headers = {'X-Run-Storlet': self.storlet_name}
if withlog is True: if withlog is True:
headers['X-Storlet-Generate-Log'] = 'True' headers['X-Storlet-Generate-Log'] = 'True'
@@ -78,21 +66,22 @@ class TestTestStorlet(unittest.TestCase):
'storletlog', 'test.log', 'storletlog', 'test.log',
None, None, None, None, None, None, None, None,
headers) headers)
assert resp_headers.get('status') == 200 self.assertEqual(resp_headers.get('status'), 200)
gf.read() gf.read()
assert resp_headers.get('status') == 200 self.assertEqual(resp_headers.get('status') == 200)
if op == 'print': if op == 'print':
assert get_response_status == 200 self.assertEqual(get_response_status, 200)
assert 'op' in get_text self.assertTrue('op' in get_text)
assert 'print' in get_text self.assertTrue('print' in get_text)
assert 'param2' in get_text self.assertTrue('param2' in get_text)
assert 'val2' in get_text self.assertTrue('val2' in get_text)
except Exception: except Exception:
get_response_status = resp_dict.get('status') get_response_status = resp_dict.get('status')
if op == 'crash': if op == 'crash':
assert get_response_status >= 500 or get_response_status == 404 self.assertTrue(get_response_status >= 500 or
get_response_status == 404)
def test_print(self): def test_print(self):
self.invokeTestStorlet("print", False) self.invokeTestStorlet("print", False)

4
tests/setup_functional_test.sh
View File

@@ -61,3 +61,7 @@ ansible-playbook -s -i deploy/hosts host_storlet_engine.yml
sudo chmod -R 777 /opt/ibm sudo chmod -R 777 /opt/ibm
echo "Running create_default_tenant playbook" echo "Running create_default_tenant playbook"
ansible-playbook -i deploy/hosts create_default_tenant.yml ansible-playbook -i deploy/hosts create_default_tenant.yml
cd -
cp cluster_config.json-sample cluster_config.json
sudo chown $USER:$USER cluster_config.json