Introducing initial cluster config

This is a baby step towards having a single cluster config that
serves:
(1) installation
(2) tests
(3) storlet mgmt
(4) various utils

At this point the functional tests and a deploy
storlet tool use a VERY initial cluster config

Change-Id: I80c4cb8c09adc2a8124f303e60489bdf1cba94b7

.gitignore

@@ -22,8 +22,9 @@ lib64
 # Installer logs
 pip-log.txt
 
-# Deploy created
+# auto created for deploy
-/Deploy/plabook/deploy
+Deploy/plabook/deploy/
+cluster_config.json
 
 # Unit test / coverage reports
 .coverage

cluster_config.json-sample

@@ -0,0 +1,11 @@
+{
+    "proxy_protocol": "HTTP",
+    "auth_protocol": "HTTP",
+    "auth_ip": "127.0.0.1",
+    "proxy_port": "80",
+    "auth_port": "5000",
+    "region": "",
+    "account": "service",
+    "user_name": "swift",
+    "password": "passw0rd"
+}

common/cluster_config_parser.py

@@ -0,0 +1,36 @@
+'''-------------------------------------------------------------------------
+Copyright IBM Corp. 2015, 2016 All Rights Reserved
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+Limitations under the License.
+-------------------------------------------------------------------------'''
+
+'''
+*VERY* initial cluster configuration file
+The intention is to have this as the single source
+for all cluster information needs such as:
+- Swift Install
+- Storlets Install
+- Tests
+- Deploy storlets tools
+- etc.
+'''
+import json
+
+
+class ClusterConfig(object):
+
+    def __init__(self, config_path):
+        conf_string = open(config_path, 'r').read()
+        self.conf = json.loads(conf_string)
+
+    def get_conf(self):
+        return self.conf

common/deploy_storlet.py

@@ -0,0 +1,63 @@
+'''-------------------------------------------------------------------------
+Copyright IBM Corp. 2015, 2016 All Rights Reserved
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+Limitations under the License.
+-------------------------------------------------------------------------'''
+
+import os
+import sys
+import zipfile
+from cluster_config_parser import ClusterConfig
+from utils import storlet_get_auth, deploy_storlet
+
+
+def list_classes(storlet_jar):
+    z = zipfile.ZipFile(storlet_jar, 'r')
+    for f in z.infolist():
+        name = f.filename
+        if name.endswith(".class"):
+            print('\t* ' + name[0:len(name) - 6].replace('/', '.'))
+    z.close()
+
+
+def usage():
+    print("Useage: deploy_storlet.py <path to conf>")
+
+
+def main():
+    if len(sys.argv) != 2:
+        usage()
+        sys.exit(-1)
+    conf = ClusterConfig(sys.argv[1]).get_conf()
+    url, token = storlet_get_auth(conf)
+    sys.stdout.write("Enter absolute path to storlet jar file: ")
+    storlet_jar = sys.stdin.readline().rstrip()
+    print("Your jar file contains the following classes:")
+    list_classes(storlet_jar)
+    sys.stdout.write("Please enter fully qualified storlet main class " +
+                     "(choose from the list above): ")
+    storlet_main_class = sys.stdin.readline().rstrip()
+    print("Please enter dependency jars (leave a blank line when you are "
+          "done):")
+    dependency_jars = []
+    dependencies = []
+    dependency_jar = sys.stdin.readline().rstrip()
+    while dependency_jar:
+        dependency_jars.append(dependency_jar)
+        dependencies.append(os.path.basename(dependency_jar))
+        dependency_jar = sys.stdin.readline().rstrip()
+
+    deploy_storlet(url, token, storlet_jar, storlet_main_class,
+                   dependency_jars, dependencies)
+
+if __name__ == "__main__":
+    main()

common/utils.py

@@ -0,0 +1,88 @@
+
+'''-------------------------------------------------------------------------
+Copyright IBM Corp. 2015, 2016 All Rights Reserved
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+Limitations under the License.
+-------------------------------------------------------------------------'''
+
+import os
+from swiftclient import client as c
+
+
+def put_local_file(url, token, container,
+                   local_path, local_file,
+                   headers=None):
+    resp = dict()
+    f = open('%s/%s' % (local_path, local_file), 'r')
+    c.put_object(url, token, container, local_file, f,
+                 headers=headers,
+                 content_type="application/octet-stream",
+                 response_dict=resp)
+    f.close()
+
+
+def put_storlet_object(url, token, storlet_name, storlet_path,
+                       dependency, main_class):
+    headers = {'X-Object-Meta-Storlet-Language': 'Java',
+               'X-Object-Meta-Storlet-Interface-Version': '1.0',
+               'X-Object-Meta-Storlet-Dependency': dependency,
+               'X-Object-Meta-Storlet-Object-Metadata': 'no',
+               'X-Object-Meta-Storlet-Main': main_class}
+    put_local_file(url, token, 'storlet', storlet_path, storlet_name, headers)
+
+
+def put_storlet_executable_dependencies(url, token, deps):
+    resp = dict()
+    for d in deps:
+        headers = {'X-Object-Meta-Storlet-Dependency-Version': '1',
+                   'X-Object-Meta-Storlet-Dependency-Permissions': '0755'}
+
+        f = open('%s' % d, 'r')
+        c.put_object(url, token, 'dependency', os.path.basename(d), f,
+                     content_type="application/octet-stream",
+                     headers=headers,
+                     response_dict=resp)
+        f.close()
+        status = resp.get('status')
+        assert (status == 200 or status == 201)
+
+
+def deploy_storlet(url, token, storlet_jar,
+                   storlet_main_class,
+                   dependencies):
+    # No need to create containers every time
+    # put_storlet_containers(url, token)
+    put_storlet_object(url, token,
+                       os.path.basename(storlet_jar),
+                       os.path.dirname(storlet_jar),
+                       ','.join(os.path.basename(x) for x in dependencies),
+                       storlet_main_class)
+
+    put_storlet_executable_dependencies(url, token, dependencies)
+
+
+def storlet_get_auth(conf):
+    auth_ip = conf['auth_ip']
+    auth_port = conf['auth_port']
+    account = conf['account']
+    user = conf['user_name']
+    passwd = conf['password']
+    os_options = {'tenant_name': account}
+    if conf['region']:
+        os_options['region_name'] = conf['region']
+    url, token = c.get_auth('http://' + auth_ip + ":" +
+                            auth_port + '/v2.0',
+                            account + ':' + user,
+                            passwd,
+                            os_options=os_options,
+                            auth_version='2.0')
+    return url, token

test-requirements.txt

@@ -7,6 +7,7 @@ hacking<0.11,>=0.10.0
 coverage>=3.6
 discover
 nose
+pexpect
 python-subunit>=0.0.18
 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
 oslotest>=1.2.0  # Apache-2.0

tests/functional/__init__.py

@@ -17,58 +17,30 @@ Limitations under the License.
 @author: gilv / cdoron / evgenyl
 '''
 
-from swiftclient import client as c
+import unittest
+from common.cluster_config_parser import ClusterConfig
+from common.utils import storlet_get_auth, deploy_storlet,\
+    put_local_file
 
-PROXY_PROTOCOL = 'HTTP'
+CONFIG_FILE = '../../cluster_config.json'
-AUTH_PROTOCOL = 'HTTP'
+PATH_TO_STORLETS = '../../StorletSamples'
-DEV_AUTH_IP = '127.0.0.1'
+BIN_DIR = 'bin'
-AUTH_IP = DEV_AUTH_IP
-PROXY_PORT = '80'
-AUTH_PORT = '5000'
-
-ACCOUNT = 'service'
-USER_NAME = 'swift'
-PASSWORD = 'passw0rd'
 
 
-def put_storlet_object(url, token, storlet_name, storlet_path,
+class StorletFunctionalTest(unittest.TestCase):
-                       dependency, main_class):
+    def setUp(self):
-    metadata = {'X-Object-Meta-Storlet-Language': 'Java',
+        conf = ClusterConfig(CONFIG_FILE).get_conf()
-                'X-Object-Meta-Storlet-Interface-Version': '1.0',
+        self.url, self.token = storlet_get_auth(conf)
-                'X-Object-Meta-Storlet-Dependency': dependency,
+        path_to_bundle = '%s/%s/%s' % (PATH_TO_STORLETS, self.storlet_dir,
-                'X-Object-Meta-Storlet-Object-Metadata': 'no',
+                                       BIN_DIR)
-                'X-Object-Meta-Storlet-Main': main_class}
+        self.deps = []
-    f = open('%s/%s' % (storlet_path, storlet_name), 'r')
+        for d in self.dep_names:
-    content_length = None
+            self.deps.append('%s/%s' % (path_to_bundle, d))
-    response = dict()
+        storlet = '%s/%s' % (path_to_bundle, self.storlet_name)
-    c.put_object(url, token, 'storlet', storlet_name, f,
-                 content_length, None, None,
-                 "application/octet-stream", metadata,
-                 None, None, None, response)
-    f.close()
-    status = response.get('status')
-    assert (status == 200 or status == 201)
 
-
+        deploy_storlet(self.url, self.token,
-def put_file_as_storlet_input_object(url, token, local_path, local_file):
+                       storlet, self.storlet_main,
-    resp = dict()
+                       self.deps)
-    f = open('%s/%s' % (local_path, local_file), 'r')
+        if self.storlet_file:
-    c.put_object(url, token, 'myobjects', local_file, f,
+            put_local_file(self.url, self.token, self.container,
-                 content_type="application/octet-stream",
+                           path_to_bundle, self.storlet_file, self.headers)
-                 response_dict=resp)
-    f.close()
-    status = resp.get('status')
-    assert (status == 200 or status == 201)
-
-
-def put_dependency(url, token, local_path_to_dep, dep_name):
-    metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1'}
-    f = open('%s/%s' % (local_path_to_dep, dep_name), 'r')
-    content_length = None
-    response = dict()
-    c.put_object(url, token, 'dependency', dep_name, f,
-                 content_length, None, None, "application/octet-stream",
-                 metadata, None, None, None, response)
-    f.close()
-    status = response.get('status')
-    assert (status == 200 or status == 201)

tests/functional/test_SLO.py

@@ -13,24 +13,12 @@ See the License for the specific language governing permissions and
 Limitations under the License.
 -------------------------------------------------------------------------'''
 
-from __init__ import ACCOUNT
-from __init__ import AUTH_IP
-from __init__ import AUTH_PORT
-from __init__ import PASSWORD
-from __init__ import put_storlet_object
-from __init__ import USER_NAME
 import json
 import os
 import random
 import string
 from swiftclient import client as c
-from test_identity_storlet import deploy_identity_storlet
+from __init__ import StorletFunctionalTest
-from test_identity_storlet import IDENTITY_STORLET_NAME
-import unittest
-
-
-SLOIDENTITY_PATH_TO_BUNDLE = '../../StorletSamples/SLOIdentityStorlet/bin'
-SLOIDENTITY_STORLET_NAME = 'sloidentitystorlet-1.0.jar'
 
 
 def create_local_chunks():
@@ -48,153 +36,6 @@ def delete_local_chunks():
         os.remove(oname)
 
 
-def put_SLO(url, token):
-    # Create temp files
-    assembly = []
-    for i in range(1, 10):
-        oname = '/tmp/slo_chunk_%d' % i
-        f = open(oname, 'r')
-        content_length = None
-        response = dict()
-        c.put_object(url, token, 'myobjects', oname, f,
-                     content_length, None, None, "application/octet-stream",
-                     None, None, None, None, response)
-        f.close()
-        status = response.get('status')
-        assert (status >= 200 and status < 300)
-
-        headers = response.get('headers')
-        segment = dict()
-        segment['path'] = 'myobjects/%s' % oname
-        segment['size_bytes'] = 1048576
-        segment['etag'] = headers['etag']
-        assembly.append(segment)
-
-    content_length = None
-    response = dict()
-    headers = {'x-object-meta-prop1': 'val1'}
-    c.put_object(url, token, 'myobjects', 'assembly', json.dumps(assembly),
-                 content_length=None, etag=None, chunk_size=None,
-                 headers=headers, query_string='multipart-manifest=put',
-                 response_dict=response)
-    status = response.get('status')
-    assert (status >= 200 and status < 300)
-
-
-def get_SLO(url, token):
-    response = dict()
-    headers, body = c.get_object(url, token, 'myobjects', 'assembly',
-                                 http_conn=None, resp_chunk_size=1048576,
-                                 query_string=None, response_dict=response,
-                                 headers=None)
-
-    i = 1
-    for chunk in body:
-        oname = '/tmp/slo_chunk_%d' % i
-        f = open(oname, 'r')
-        file_content = f.read()
-        # print '%s    %s' % (chunk[:10], file_content[:10])
-        # print '%d    %d' % (len(chunk), len(file_content))
-        assert(chunk == file_content)
-        f.close()
-        i = i + 1
-
-
-def compare_slo_to_chunks(body):
-    i = 1
-    for chunk in body:
-        if chunk:
-            if i < 10:
-                oname = '/tmp/slo_chunk_%d' % i
-                f = open(oname, 'r')
-                file_content = f.read()
-                # print '%s    %s' % (chunk[:10], file_content[:10])
-                # print '%d    %d' % (len(chunk), len(file_content))
-                assert(chunk == file_content)
-                f.close()
-                i = i + 1
-            else:
-                aux_content = ''
-                for j in range(1, 4):
-                    oname = '/tmp/aux_file%d' % j
-                    f = open(oname, 'r')
-                    aux_content += f.read()
-                    f.close()
-                assert(chunk == aux_content)
-
-
-def invoke_identity_on_get_SLO(url, token):
-    metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
-    response = dict()
-    headers, body = c.get_object(url, token,
-                                 'myobjects', 'assembly',
-                                 query_string=None,
-                                 response_dict=response,
-                                 resp_chunk_size=1048576,
-                                 headers=metadata)
-    compare_slo_to_chunks(body)
-
-
-def invoke_identity_on_get_SLO_double(url, token):
-    metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
-    response = dict()
-    headers, body = c.get_object(url, token,
-                                 'myobjects',
-                                 'assembly',
-                                 query_string='double=true',
-                                 response_dict=response,
-                                 resp_chunk_size=2048,
-                                 headers=metadata)
-
-    i = 1
-    oname = '/tmp/slo_chunk_%d' % i
-    f = open(oname, 'r')
-    file_content = f.read()
-
-    j = 0  # Count chunks in file 1...1024
-    for chunk in body:
-        file_fragment = file_content[j * 1024:(j + 1) * 1024]
-        chunk_framgment_low = chunk[0:1024]
-        chunk_framgment_high = chunk[1024:2048]
-        assert(chunk_framgment_low == file_fragment)
-        assert(chunk_framgment_high == file_fragment)
-        j = j + 1
-        if j == 1024:
-            i = i + 1
-            if i == 10:
-                break
-            f.close()
-            oname = '/tmp/slo_chunk_%d' % i
-            f = open(oname, 'r')
-            file_content = f.read()
-            j = 0
-    assert i == 10
-
-
-def invoke_identity_on_partial_get_SLO(url, token):
-    metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
-    for i in range(5):
-        response = dict()
-        headers, body = c.get_object(url, token,
-                                     'myobjects',
-                                     'assembly',
-                                     query_string=None,
-                                     response_dict=response,
-                                     resp_chunk_size=1048576,
-                                     headers=metadata)
-
-        j = 1
-        for chunk in body:
-            j = j + 1
-            if j == 5:
-                break
-
-# def delete_files():
-#     for i in range(1,4):
-#         fname = '/tmp/aux_file%d' % i
-#         os.remove(fname)
-
-
 def create_container(url, token, name):
     response = dict()
     c.put_container(url, token, name, headers=None, response_dict=response)
@@ -202,44 +43,112 @@ def create_container(url, token, name):
     assert (status >= 200 or status < 300)
 
 
-def deploy_sloidentity_storlet(url, token):
+class TestSLO(StorletFunctionalTest):
-    response = dict()
-    c.put_container(url, token, 'mysloobject', None, None, response)
-    status = response.get('status')
-    assert (status >= 200 or status < 300)
-
-    put_storlet_object(url, token,
-                       SLOIDENTITY_STORLET_NAME,
-                       SLOIDENTITY_PATH_TO_BUNDLE,
-                       '',
-                       'com.ibm.storlet.sloidentity.SLOIdentityStorlet')
-
-
-class TestSLO(unittest.TestCase):
     def setUp(self):
-        os_options = {'tenant_name': ACCOUNT}
+        self.storlet_dir = 'IdentityStorlet'
-        self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+        self.storlet_name = 'identitystorlet-1.0.jar'
-                                          + "/v2.0", ACCOUNT + ":" + USER_NAME,
+        self.storlet_main = 'com.ibm.storlet.identity.IdentityStorlet'
-                                          PASSWORD, os_options=os_options,
+        self.storlet_log = 'identitystorlet-1.0.log'
-                                          auth_version="2.0")
+        self.headers = None
+        self.storlet_file = ''
+        self.container = 'myobjects'
+        self.dep_names = []
+        super(TestSLO, self).setUp()
+
         create_container(self.url, self.token, 'myobjects')
         create_container(self.url, self.token, 'container1')
         create_container(self.url, self.token, 'container2')
         create_container(self.url, self.token, 'container3')
         create_local_chunks()
-        put_SLO(self.url, self.token)
+        self.put_SLO()
-        get_SLO(self.url, self.token)
+        self.get_SLO()
-        deploy_identity_storlet(self.url, self.token)
 
     def tearDown(self):
         delete_local_chunks()
 
+    def get_SLO(self):
+        response = dict()
+        headers, body = c.get_object(self.url, self.token,
+                                     self.container, 'assembly',
+                                     http_conn=None, resp_chunk_size=1048576,
+                                     query_string=None, response_dict=response,
+                                     headers=None)
+
+        i = 1
+        for chunk in body:
+            oname = '/tmp/slo_chunk_%d' % i
+            f = open(oname, 'r')
+            file_content = f.read()
+            # print '%s    %s' % (chunk[:10], file_content[:10])
+            # print '%d    %d' % (len(chunk), len(file_content))
+            self.assertEqual(chunk, file_content)
+            f.close()
+            i = i + 1
+
+    def put_SLO(self):
+        # Create temp files
+        assembly = []
+        for i in range(1, 10):
+            oname = '/tmp/slo_chunk_%d' % i
+            f = open(oname, 'r')
+            content_length = None
+            response = dict()
+            c.put_object(self.url, self.token,
+                         self.container, oname, f,
+                         content_length, None, None,
+                         "application/octet-stream",
+                         None, None, None, None, response)
+            f.close()
+            status = response.get('status')
+            self.assertTrue(status >= 200 and status < 300)
+
+            headers = response.get('headers')
+            segment = dict()
+            segment['path'] = 'myobjects/%s' % oname
+            segment['size_bytes'] = 1048576
+            segment['etag'] = headers['etag']
+            assembly.append(segment)
+
+        content_length = None
+        response = dict()
+        headers = {'x-object-meta-prop1': 'val1'}
+        c.put_object(self.url, self.token, self.container,
+                     'assembly', json.dumps(assembly),
+                     content_length=None, etag=None, chunk_size=None,
+                     headers=headers, query_string='multipart-manifest=put',
+                     response_dict=response)
+        status = response.get('status')
+        self.assertTrue(status >= 200 and status < 300)
+
+    def compare_slo_to_chunks(self, body):
+        i = 1
+        for chunk in body:
+            if chunk:
+                if i < 10:
+                    oname = '/tmp/slo_chunk_%d' % i
+                    f = open(oname, 'r')
+                    file_content = f.read()
+                    # print '%s    %s' % (chunk[:10], file_content[:10])
+                    # print '%d    %d' % (len(chunk), len(file_content))
+                    self.assertEqual(chunk, file_content)
+                    f.close()
+                    i = i + 1
+                else:
+                    aux_content = ''
+                    for j in range(1, 4):
+                        oname = '/tmp/aux_file%d' % j
+                        f = open(oname, 'r')
+                        aux_content += f.read()
+                        f.close()
+                    self.ssertEqual(chunk, aux_content)
+
     def test_get_SLO(self):
-        invoke_identity_on_get_SLO(self.url, self.token)
+        headers = {'X-Run-Storlet': self.storlet_name}
-
+        response = dict()
-    # YM comment out 2 lines - temporary only!
+        headers, body = c.get_object(self.url, self.token,
-        # progress_msg("Invoking storlet on SLO in GET with double")
+                                     'myobjects', 'assembly',
-    # invoke_identity_on_get_SLO_double(url, token)
+                                     query_string=None,
-
+                                     response_dict=response,
-    # progress_msg("Invoking storlet on SLO in partial GET")
+                                     resp_chunk_size=1048576,
-    # invoke_identity_on_partial_get_SLO(url, token)
+                                     headers=headers)
+        self.compare_slo_to_chunks(body)

tests/functional/test_deploy_storlet.py

@@ -0,0 +1,49 @@
+'''-------------------------------------------------------------------------
+Copyright IBM Corp. 2015, 2015 All Rights Reserved
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+Limitations under the License.
+-------------------------------------------------------------------------'''
+
+import unittest
+import pexpect
+from __init__ import PATH_TO_STORLETS, BIN_DIR
+
+DEPLOY_STORLET_PATH = '../../common/deploy_storlet.py'
+CONF_PATH = '../../cluster_config.json'
+EXECDEP_STORLET_PATH = '%s/%s/%s' % (PATH_TO_STORLETS,
+                                     'ExecDepStorlet',
+                                     BIN_DIR)
+EXECDEP_STORLET_PATH = '%s/%s' % (EXECDEP_STORLET_PATH,
+                                  'execdepstorlet-1.0.jar')
+EXECDEP_STORLET_DEP_PATH = '%s/%s' % (EXECDEP_STORLET_PATH,
+                                      'get42')
+
+
+class TestExecDepStorlet(unittest.TestCase):
+
+    def test_deploy_storlet_util(self):
+        child = pexpect.spawn('python %s %s' % (DEPLOY_STORLET_PATH,
+                                                CONF_PATH))
+        try:
+            child.expect('Enter absolute path to storlet jar file.*:',
+                         timeout=1)
+            child.sendline(EXECDEP_STORLET_PATH)
+            child.expect('com.ibm.storlet.execdep.ExecDepStorlet',
+                         timeout=1)
+            child.expect('Please enter fully qualified storlet main class.*',
+                         timeout=1)
+            child.sendline('com.ibm.storlet.execdep.ExecDepStorlet')
+            child.expect('Please enter dependency.*', timeout=1)
+            child.sendline(EXECDEP_STORLET_DEP_PATH)
+            child.expect('\n')
+        except Exception as err:
+            self.fail('Unexpected exception %s' % err)

tests/functional/test_execdep_storlet.py

@@ -13,73 +13,31 @@ See the License for the specific language governing permissions and
 Limitations under the License.
 -------------------------------------------------------------------------'''
 
-from __init__ import ACCOUNT
-from __init__ import AUTH_IP
-from __init__ import AUTH_PORT
-from __init__ import PASSWORD
-from __init__ import put_file_as_storlet_input_object
-from __init__ import put_storlet_object
-from __init__ import USER_NAME
 from swiftclient import client as c
-import unittest
+from __init__ import StorletFunctionalTest
-
-EXECDEP_PATH_TO_BUNDLE = '../../StorletSamples/ExecDepStorlet/bin/'
-EXECDEP_STORLET_NAME = 'execdepstorlet-1.0.jar'
-EXECDEP_STORLET_LOG_NAME = 'execdepstorlet-1.0.log'
-EXECDEP_JUNK_FILE = 'junk.txt'
-EXECDEP_DEPS_NAMES = ['get42']
 
 
-class TestExexDepStorlet(unittest.TestCase):
+class TestExecDepStorlet(StorletFunctionalTest):
     def setUp(self):
-        os_options = {'tenant_name': ACCOUNT}
+        self.storlet_dir = 'ExecDepStorlet'
-        self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+        self.storlet_name = 'execdepstorlet-1.0.jar'
-                                          + "/v2.0", ACCOUNT + ":" + USER_NAME,
+        self.storlet_main = 'com.ibm.storlet.execdep.ExecDepStorlet'
-                                          PASSWORD, os_options=os_options,
+        self.storlet_log = 'execdepstorlet-1.0.log'
-                                          auth_version="2.0")
+        self.headers = None
+        self.storlet_file = 'junk.txt'
+        self.container = 'myobjects'
+        self.dep_names = ['get42']
+        super(TestExecDepStorlet, self).setUp()
 
-    def put_storlet_executable_dependencies(self):
+    def test_execdep(self):
-        resp = dict()
+        headers = {'X-Run-Storlet': self.storlet_name}
-        for d in EXECDEP_DEPS_NAMES:
-            metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1',
-                        'X-Object-Meta-Storlet-Dependency-Permissions': '0755'}
-
-            f = open('%s/%s' % (EXECDEP_PATH_TO_BUNDLE, d), 'r')
-            c.put_object(self.url, self.token, 'dependency', d, f,
-                         content_type="application/octet-stream",
-                         headers=metadata,
-                         response_dict=resp)
-            f.close()
-            status = resp.get('status')
-            assert (status == 200 or status == 201)
-
-    def deploy_storlet(self):
-        # No need to create containers every time
-        # put_storlet_containers(url, token)
-        put_storlet_object(self.url, self.token,
-                           EXECDEP_STORLET_NAME,
-                           EXECDEP_PATH_TO_BUNDLE,
-                           ','.join(str(x) for x in EXECDEP_DEPS_NAMES),
-                           'com.ibm.storlet.execdep.ExecDepStorlet')
-        self.put_storlet_executable_dependencies()
-        put_file_as_storlet_input_object(self.url,
-                                         self.token,
-                                         EXECDEP_PATH_TO_BUNDLE,
-                                         EXECDEP_JUNK_FILE)
-
-    def invoke_storlet(self):
-        metadata = {'X-Run-Storlet': EXECDEP_STORLET_NAME}
         resp = dict()
         resp_headers, gf = c.get_object(self.url, self.token,
                                         'myobjects',
-                                        EXECDEP_JUNK_FILE,
+                                        self.storlet_file,
                                         response_dict=resp,
-                                        headers=metadata)
+                                        headers=headers)
 
-        assert 'x-object-meta-depend-ret-code' in resp_headers
+        self.assertTrue('x-object-meta-depend-ret-code' in resp_headers)
-        assert resp_headers['x-object-meta-depend-ret-code'] == '42'
+        self.assertTrue(resp_headers['x-object-meta-depend-ret-code'] == '42')
-        assert resp['status'] == 200
+        self.assertEqual(resp['status'], 200)
-
-    def test_execdep(self):
-        self.deploy_storlet()
-        self.invoke_storlet()

tests/functional/test_half_storlet.py

@@ -13,126 +13,99 @@ See the License for the specific language governing permissions and
 Limitations under the License.
 -------------------------------------------------------------------------'''
 
-from __init__ import ACCOUNT
-from __init__ import AUTH_IP
-from __init__ import AUTH_PORT
-from __init__ import PASSWORD
-from __init__ import put_storlet_object
-from __init__ import USER_NAME
 import random
 import string
 from swiftclient import client as c
-import unittest
+from __init__ import StorletFunctionalTest
 
 
-# Test Constants
+class TestHalfIdentityStorlet(StorletFunctionalTest):
-HALF_PATH_TO_BUNDLE = '../../StorletSamples/HalfStorlet/bin/'
-HALF_STORLET_NAME = 'halfstorlet-1.0.jar'
-HALF_SOURCE_FILE = 'source.txt'
-
-
-def put_storlet_input_object(url, token):
-    resp = dict()
-    metadata = {'X-Object-Meta-Testkey': 'tester'}
-    f = open('%s/%s' % (HALF_PATH_TO_BUNDLE, HALF_SOURCE_FILE), 'r')
-    c.put_object(url, token, 'myobjects', HALF_SOURCE_FILE, f,
-                 content_type="application/octet-stream",
-                 headers=metadata,
-                 response_dict=resp)
-    f.close()
-    status = resp.get('status')
-    assert (status == 200 or status == 201)
-
-
-def deploy_storlet(url, token):
-    # No need to create containers every time
-    # put_storlet_containers(url, token)
-    put_storlet_object(url, token,
-                       HALF_STORLET_NAME,
-                       HALF_PATH_TO_BUNDLE,
-                       '',
-                       'com.ibm.storlet.half.HalfStorlet')
-    put_storlet_input_object(url, token)
-
-
-def invoke_storlet(url, token, op, params=None, global_params=None,
-                   headers=None):
-    if params is not None:
-        querystring = ''
-        for key in params:
-            querystring += '%s=%s,' % (key, params[key])
-        querystring = querystring[:-1]
-    else:
-        querystring = None
-
-    metadata = {'X-Run-Storlet': HALF_STORLET_NAME}
-    if headers:
-        metadata.update(headers)
-
-    if op == 'GET':
-        # Get original object
-        original_headers, original_content = \
-            c.get_object(url, token, 'myobjects', HALF_SOURCE_FILE,
-                         response_dict=dict())
-        # print original_headers
-        file_length = int(original_headers['content-length'])
-        processed_headers, returned_content = \
-            c.get_object(url, token, 'myobjects', HALF_SOURCE_FILE,
-                         query_string=querystring, response_dict=dict(),
-                         headers=metadata, resp_chunk_size=file_length)
-        processed_content = ''
-        for chunk in returned_content:
-            if chunk:
-                processed_content += chunk
-
-        assert(original_headers['X-Object-Meta-Testkey'.lower()] ==
-               processed_headers['X-Object-Meta-Testkey'.lower()])
-        return processed_content
-
-    if op == 'PUT':
-        # PUT a random file
-        response = dict()
-        uploaded_content = ''.join(random.choice(string.ascii_uppercase +
-                                   string.digits) for _ in range(1024))
-        random_md = ''.join(random.choice(string.ascii_uppercase +
-                            string.digits) for _ in range(32))
-        # content_length = 1024
-        content_length = None
-        headers = {'X-Run-Storlet': HALF_STORLET_NAME,
-                   'X-Object-Meta-Testkey': random_md}
-        c.put_object(url, token, 'myobjects', 'half_random_source',
-                     uploaded_content, content_length, None, None,
-                     "application/octet-stream", headers, None, None,
-                     querystring, response)
-        resp_headers, saved_content = c.get_object(url, token, 'myobjects',
-                                                   'half_random_source',
-                                                   response_dict=dict())
-
-        if params is not None and params.get('double', None) == 'true':
-            assert(uploaded_content == saved_content[:1024])
-            assert(uploaded_content == saved_content[1024:])
-        else:
-            assert(uploaded_content == saved_content)
-
-        if params is not None and params.get('execute', None) is not None:
-            assert(resp_headers['X-Object-Meta-Execution result'.lower()] ==
-                   '42')
-
-        assert(resp_headers['X-Object-Meta-Testkey'.lower()] == random_md)
-
-
-class TestHalfIdentityStorlet(unittest.TestCase):
     def setUp(self):
-        os_options = {'tenant_name': ACCOUNT}
+        self.storlet_dir = 'HalfStorlet'
-        self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+        self.storlet_name = 'halfstorlet-1.0.jar'
-                                          + "/v2.0", ACCOUNT + ":" + USER_NAME,
+        self.storlet_main = 'com.ibm.storlet.half.HalfStorlet'
-                                          PASSWORD, os_options=os_options,
+        self.storlet_log = ''
-                                          auth_version='2.0')
+        self.headers = {'X-Object-Meta-Testkey': 'tester'}
-        deploy_storlet(self.url, self.token)
+        self.storlet_file = 'source.txt'
+        self.container = 'myobjects'
+        self.dep_names = []
+        super(TestHalfIdentityStorlet, self).setUp()
+
+    def invoke_storlet(self, op, params=None, global_params=None,
+                       headers=None):
+        if params is not None:
+            querystring = ''
+            for key in params:
+                querystring += '%s=%s,' % (key, params[key])
+            querystring = querystring[:-1]
+        else:
+            querystring = None
+
+        req_headers = {'X-Run-Storlet': self.storlet_name}
+        if headers:
+            req_headers.update(headers)
+
+        if op == 'GET':
+            # Get original object
+            original_h, original_c = \
+                c.get_object(self.url, self.token, 'myobjects',
+                             self.storlet_file,
+                             response_dict=dict())
+            # print original_headers
+            file_length = int(original_h['content-length'])
+            processed_h, returned_c = \
+                c.get_object(self.url, self.token, 'myobjects',
+                             self.storlet_file,
+                             query_string=querystring, response_dict=dict(),
+                             headers=req_headers, resp_chunk_size=file_length)
+            processed_c = ''
+            for chunk in returned_c:
+                if chunk:
+                    processed_c += chunk
+
+            self.assertEqual(original_h['X-Object-Meta-Testkey'.lower()],
+                             processed_h['X-Object-Meta-Testkey'.lower()])
+            return processed_c
+
+        if op == 'PUT':
+            # PUT a random file
+            response = dict()
+            uploaded_content = ''.join(random.choice(string.ascii_uppercase +
+                                       string.digits) for _ in range(1024))
+            random_md = ''.join(random.choice(string.ascii_uppercase +
+                                string.digits) for _ in range(32))
+            # content_length = 1024
+            content_length = None
+            headers = {'X-Run-Storlet': self.storlet_name,
+                       'X-Object-Meta-Testkey': random_md}
+            c.put_object(self.url, self.token, self.container,
+                         'half_random_source',
+                         uploaded_content, content_length, None, None,
+                         "application/octet-stream", headers, None, None,
+                         querystring, response)
+            resp_headers, saved_content = c.get_object(self.url, self.token,
+                                                       'myobjects',
+                                                       'half_random_source',
+                                                       response_dict=dict())
+
+            if params is not None and params.get('double', None) == 'true':
+                self.assertEqual(uploaded_content, saved_content[:1024])
+                self.assertEqual(uploaded_content, saved_content[1024:])
+            else:
+                self.assertEqual(uploaded_content, saved_content)
+
+            if params is not None and params.get('execute', None) is not None:
+                self.assertEqual(
+                    resp_headers['X-Object-Meta-Execution result'.lower()],
+                    '42')
+
+            self.assertEqual(resp_headers['X-Object-Meta-Testkey'.lower()],
+                             random_md)
 
     def test_get(self):
-        assert (invoke_storlet(self.url, self.token, 'GET') == 'acegikmn')
+        res = self.invoke_storlet('GET')
+        self.assertEqual(res, 'acegikmn')
 
     def test_get_range(self):
-        assert (invoke_storlet(self.url, self.token, 'GET',
+        res = self.invoke_storlet('GET', headers={'range': 'bytes=5-10'})
-                headers={'range': 'bytes=5-10'}) == 'fhj')
+        self.assertEqual(res, 'fhj')

tests/functional/test_identity_storlet.py

@@ -13,182 +13,130 @@ See the License for the specific language governing permissions and
 Limitations under the License.
 -------------------------------------------------------------------------'''
 
-from __init__ import ACCOUNT
-from __init__ import AUTH_IP
-from __init__ import AUTH_PORT
-from __init__ import PASSWORD
-from __init__ import put_storlet_object
-from __init__ import USER_NAME
 import os
 import random
 import string
 from swiftclient import client as c
-import unittest
+from __init__ import StorletFunctionalTest
 
 
-# Test Constants
+class TestIdentityStorlet(StorletFunctionalTest):
-IDENTITY_PATH_TO_BUNDLE = '../../StorletSamples/IdentityStorlet/bin/'
-IDENTITY_STORLET_NAME = 'identitystorlet-1.0.jar'
-IDENTITY_STORLET_LOG_NAME = 'identitystorlet-1.0.log'
-IDENTITY_SOURCE_FILE = 'source.txt'
-IDENTITY_DEPS_NAMES = ['get42']
-
-
-def put_storlet_executable_dependencies(url, token):
-    resp = dict()
-    for d in IDENTITY_DEPS_NAMES:
-        metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1',
-                    'X-Object-Meta-Storlet-Dependency-Permissions': '0755'}
-
-        f = open('%s/%s' % (IDENTITY_PATH_TO_BUNDLE, d), 'r')
-        c.put_object(url, token, 'dependency', d, f,
-                     content_type="application/octet-stream",
-                     headers=metadata,
-                     response_dict=resp)
-        f.close()
-        status = resp.get('status')
-        assert (status == 200 or status == 201)
-
-
-def put_storlet_input_object(url, token):
-    resp = dict()
-    metadata = {'X-Object-Meta-Testkey': 'tester'}
-    f = open('%s/%s' % (IDENTITY_PATH_TO_BUNDLE, IDENTITY_SOURCE_FILE), 'r')
-    c.put_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE, f,
-                 content_type="application/octet-stream",
-                 headers=metadata,
-                 response_dict=resp)
-    f.close()
-    status = resp.get('status')
-    assert (status == 200 or status == 201)
-
-
-def deploy_identity_storlet(url, token):
-    # No need to create containers every time
-    # put_storlet_containers(url, token)
-    put_storlet_object(url, token,
-                       IDENTITY_STORLET_NAME,
-                       IDENTITY_PATH_TO_BUNDLE,
-                       ','.join(str(x) for x in IDENTITY_DEPS_NAMES),
-                       'com.ibm.storlet.identity.IdentityStorlet')
-    put_storlet_executable_dependencies(url, token)
-    put_storlet_input_object(url, token)
-
-
-def invoke_storlet_on_1GB_file(url, token):
-    GBFile = open('/tmp/1GB_file', 'w')
-    for _ in range(128):
-        uploaded_content = ''.join('1' for _ in range(8 * 1024 * 1024))
-        GBFile.write(uploaded_content)
-    GBFile.close()
-
-    headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
-    GBFile = open('/tmp/1GB_file', 'r')
-    response = dict()
-    c.put_object(url, token, 'myobjects', '1GBFile', GBFile,
-                 1024 * 1024 * 1024, None, None, "application/octet-stream",
-                 headers, None, None, None, response)
-    status = response.get('status')
-    assert (status == 200 or status == 201)
-    os.remove('/tmp/1GB_file')
-
-
-def invoke_storlet(url, token, op, params=None, global_params=None):
-    if params is not None:
-        querystring = ''
-        for key in params:
-            querystring += '%s=%s,' % (key, params[key])
-        querystring = querystring[:-1]
-    else:
-        querystring = None
-
-    metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
-    if op == 'GET':
-        # Get original object
-        original_headers, original_content = \
-            c.get_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE,
-                         response_dict=dict())
-        # print original_headers
-        file_length = int(original_headers['content-length'])
-        processed_headers, returned_content = \
-            c.get_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE,
-                         query_string=querystring, response_dict=dict(),
-                         headers=metadata, resp_chunk_size=file_length)
-        processed_content = ''
-        for chunk in returned_content:
-            if chunk:
-                processed_content += chunk
-
-        if params is not None and params.get('execute', None) is not None:
-            assert(processed_headers['X-Object-Meta-Execution result'.lower()]
-                   == '42')
-        if params is not None and params.get('double', None) == 'true':
-            assert(original_content == processed_content[:file_length])
-            assert(original_content == processed_content[file_length:])
-        else:
-            assert(original_content == processed_content)
-        assert(original_headers['X-Object-Meta-Testkey'.lower()] ==
-               processed_headers['X-Object-Meta-Testkey'.lower()])
-
-    if op == 'PUT':
-        # PUT a random file
-        response = dict()
-        uploaded_content = ''.join(random.choice(string.ascii_uppercase +
-                                   string.digits) for _ in range(1024))
-        random_md = ''.join(random.choice(string.ascii_uppercase +
-                            string.digits) for _ in range(32))
-        content_length = None
-        headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME,
-                   'X-Object-Meta-Testkey': random_md}
-        c.put_object(url, token, 'myobjects', 'identity_random_source',
-                     uploaded_content, content_length, None, None,
-                     "application/octet-stream", headers, None, None,
-                     querystring, response)
-        resp_headers, saved_content = c.get_object(url, token,
-                                                   'myobjects',
-                                                   'identity_random_source',
-                                                   response_dict=dict())
-
-        if params is not None and params.get('double', None) == 'true':
-            assert(uploaded_content == saved_content[:1024])
-            assert(uploaded_content == saved_content[1024:])
-        else:
-            assert(uploaded_content == saved_content)
-
-        if params is not None and params.get('execute', None) is not None:
-            assert(resp_headers['X-Object-Meta-Execution result'.lower()] ==
-                   '42')
-
-        assert(resp_headers['X-Object-Meta-Testkey'.lower()] == random_md)
-
-
-class TestIdentityStorlet(unittest.TestCase):
     def setUp(self):
-        os_options = {'tenant_name': ACCOUNT}
+        self.storlet_dir = 'IdentityStorlet'
-        self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+        self.storlet_name = 'identitystorlet-1.0.jar'
-                                          + "/v2.0", ACCOUNT + ":" + USER_NAME,
+        self.storlet_main = 'com.ibm.storlet.identity.IdentityStorlet'
-                                          PASSWORD, os_options=os_options,
+        self.storlet_log = 'identitystorlet-1.0.log'
-                                          auth_version="2.0")
+        self.headers = {'X-Object-Meta-Testkey': 'tester'}
+        self.storlet_file = 'source.txt'
+        self.container = 'myobjects'
+        self.dep_names = ['get42']
+        super(TestIdentityStorlet, self).setUp()
 
-        deploy_identity_storlet(self.url, self.token)
+    def invoke_storlet(self, op, params=None, global_params=None):
+        if params is not None:
+            querystring = ''
+            for key in params:
+                querystring += '%s=%s,' % (key, params[key])
+            querystring = querystring[:-1]
+        else:
+            querystring = None
+
+        headers = {'X-Run-Storlet': self.storlet_name}
+        if op == 'GET':
+            # Get original object
+            original_h, original_c = c.get_object(self.url, self.token,
+                                                  'myobjects',
+                                                  self.storlet_file,
+                                                  response_dict=dict())
+            # print original_headers
+            file_length = int(original_h['content-length'])
+            processed_h, returned_c = c.get_object(
+                self.url, self.token,
+                self.container, self.storlet_file,
+                query_string=querystring, response_dict=dict(),
+                headers=headers, resp_chunk_size=file_length)
+            processed_c = ''
+            for chunk in returned_c:
+                if chunk:
+                    processed_c += chunk
+
+            if params is not None and params.get('execute', None) is not None:
+                mdv = processed_h['X-Object-Meta-Execution result'.lower()]
+                self.assertEqual(mdv, '42')
+            if params is not None and params.get('double', None) == 'true':
+                self.assertEqual(original_c, processed_c[:file_length])
+                self.assertEqual(original_c, processed_c[file_length:])
+            else:
+                self.assertEqual(original_c, processed_c)
+            self.assertEqual(original_h['X-Object-Meta-Testkey'.lower()],
+                             processed_h['X-Object-Meta-Testkey'.lower()])
+
+        if op == 'PUT':
+            # PUT a random file
+            response = dict()
+            uploaded_c = ''.join(random.choice(string.ascii_uppercase +
+                                 string.digits) for _ in range(1024))
+            random_md = ''.join(random.choice(string.ascii_uppercase +
+                                string.digits) for _ in range(32))
+            content_length = None
+            headers = {'X-Run-Storlet': self.storlet_name,
+                       'X-Object-Meta-Testkey': random_md}
+            c.put_object(self.url, self.token, self.container,
+                         'identity_random_source',
+                         uploaded_c, content_length, None, None,
+                         "application/octet-stream", headers, None, None,
+                         querystring, response)
+            resp_headers, saved_c = c.get_object(self.url, self.token,
+                                                 'myobjects',
+                                                 'identity_random_source',
+                                                 response_dict=dict())
+
+            if params is not None and params.get('double', None) == 'true':
+                assert(uploaded_c == saved_c[:1024])
+                assert(uploaded_c == saved_c[1024:])
+            else:
+                assert(uploaded_c == saved_c)
+
+            if params is not None and params.get('execute', None) is not None:
+                mdv = resp_headers['X-Object-Meta-Execution result'.lower()]
+                self.assertEqual(mdv, '42')
+
+            self.assertEqual(resp_headers['X-Object-Meta-Testkey'.lower()],
+                             random_md)
+
+    def test_put_1GB_file(self):
+        GBFile = open('/tmp/1GB_file', 'w')
+        for _ in range(128):
+            uploaded_content = ''.join('1' for _ in range(8 * 1024 * 1024))
+            GBFile.write(uploaded_content)
+        GBFile.close()
+
+        headers = {'X-Run-Storlet': self.storlet_name}
+        GBFile = open('/tmp/1GB_file', 'r')
+        response = dict()
+        c.put_object(self.url, self.token,
+                     self.container, '1GBFile', GBFile,
+                     1024 * 1024 * 1024, None, None,
+                     "application/octet-stream",
+                     headers, None, None, None, response)
+        status = response.get('status')
+        self.assertTrue(status in [200, 201])
+        os.remove('/tmp/1GB_file')
 
     def test_put(self):
-        invoke_storlet(self.url, self.token, 'PUT')
+        self.invoke_storlet('PUT')
-
-    def test_put_1GB(self):
-        invoke_storlet_on_1GB_file(self.url, self.token)
 
     def test_put_execute(self):
-        invoke_storlet(self.url, self.token, 'PUT', {'execute': 'true'})
+        self.invoke_storlet('PUT', {'execute': 'true'})
 
     def test_put_double(self):
-        invoke_storlet(self.url, self.token, 'PUT', {'double': 'true'})
+        self.invoke_storlet('PUT', {'double': 'true'})
 
     def test_get(self):
-        invoke_storlet(self.url, self.token, 'GET')
+        self.invoke_storlet('GET')
 
     def test_get_double(self):
-        invoke_storlet(self.url, self.token, 'GET', {'double': 'true'})
+        self.invoke_storlet('GET', {'double': 'true'})
 
     def test_get_execute(self):
-        invoke_storlet(self.url, self.token, 'GET', {'execute': 'true'})
+        self.invoke_storlet('GET', {'execute': 'true'})

tests/functional/test_metadata_storlet.py

@@ -13,92 +13,55 @@ See the License for the specific language governing permissions and
 Limitations under the License.
 ----------------------------------------------------------------'''
 
-from __init__ import ACCOUNT
-from __init__ import AUTH_IP
-from __init__ import AUTH_PORT
-from __init__ import PASSWORD
-from __init__ import put_storlet_object
-from __init__ import USER_NAME
 from swiftclient import client as c
-import unittest
+from __init__ import StorletFunctionalTest
-
-# Test Constants
-PATH_TO_BUNDLE = '../../StorletSamples/TestMetadataStorlet/bin/'
-STORLET_NAME = 'testmetadatastorlet-1.0.jar'
-STORLET_LOG_NAME = 'testmetadatastorlet-1.0.log'
-SOURCE_FILE = 'source.txt'
 
 
-def put_storlet_input_object(url, token):
+class TestMetadataStorlet(StorletFunctionalTest):
-    resp = dict()
-    metadata = {'X-Object-Meta-key1': '1',
-                'X-Object-Meta-key2': '2',
-                'X-Object-Meta-key3': '3',
-                'X-Object-Meta-key4': '4',
-                'X-Object-Meta-key5': '5',
-                'X-Object-Meta-key6': '6',
-                'X-Object-Meta-key7': '7',
-                'X-Object-Meta-key8': '8',
-                'X-Object-Meta-key9': '9',
-                'X-Object-Meta-key10': '10'}
-    f = open('%s/%s' % (PATH_TO_BUNDLE, SOURCE_FILE), 'r')
-    c.put_object(url, token, 'myobjects', SOURCE_FILE, f,
-                 content_type="application/octet-stream",
-                 headers=metadata,
-                 response_dict=resp)
-    f.close()
-    status = resp.get('status')
-    assert (status == 200 or status == 201)
-
-
-def deploy_storlet(url, token):
-    # No need to create containers every time
-    # put_storlet_containers(url, token)
-    put_storlet_object(url, token,
-                       STORLET_NAME,
-                       PATH_TO_BUNDLE,
-                       '',
-                       'com.ibm.storlet.testmetadatastorlet.MetadataStorlet')
-    put_storlet_input_object(url, token)
-
-
-def invoke_storlet(url, token, op, params=None, global_params=None):
-    if params is not None:
-        querystring = ''
-        for key in params:
-            querystring += '%s=%s,' % (key, params[key])
-        querystring = querystring[:-1]
-    else:
-        querystring = None
-
-    metadata = {'X-Run-Storlet': STORLET_NAME}
-    if op == 'GET':
-        # Get original object
-        original_headers, original_content = \
-            c.get_object(url, token, 'myobjects', SOURCE_FILE,
-                         response_dict=dict(), headers=metadata)
-        assert(original_headers['X-Object-Meta-key1'.lower()] == '1')
-        assert(original_headers['X-Object-Meta-key2'.lower()] == '2')
-        assert(original_headers['X-Object-Meta-key3'.lower()] == '3')
-        assert(original_headers['X-Object-Meta-key4'.lower()] == '4')
-        assert(original_headers['X-Object-Meta-key5'.lower()] == '5')
-        assert(original_headers['X-Object-Meta-key6'.lower()] == '6')
-        assert(original_headers['X-Object-Meta-key7'.lower()] == '7')
-        assert(original_headers['X-Object-Meta-key8'.lower()] == '8')
-        assert(original_headers['X-Object-Meta-key9'.lower()] == '9')
-        assert(original_headers['X-Object-Meta-key10'.lower()] == '10')
-        assert(original_headers['X-Object-Meta-override_key'.lower()] ==
-               'new_value')
-
-
-class TestMetadataStorlet(unittest.TestCase):
     def setUp(self):
-        os_options = {'tenant_name': ACCOUNT}
+        self.storlet_dir = 'TestMetadataStorlet'
-        self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+        self.storlet_name = 'testmetadatastorlet-1.0.jar'
-                                          + "/v2.0", ACCOUNT + ":" + USER_NAME,
+        self.storlet_main = ('com.ibm.storlet.testmetadatastorlet'
-                                          PASSWORD, os_options=os_options,
+                             '.MetadataStorlet')
-                                          auth_version="2.0")
+        self.storlet_log = 'testmetadatastorlet-1.0.log'
-        deploy_storlet(self.url, self.token)
+        self.headers = {'X-Object-Meta-key1': '1',
+                        'X-Object-Meta-key2': '2',
+                        'X-Object-Meta-key3': '3',
+                        'X-Object-Meta-key4': '4',
+                        'X-Object-Meta-key5': '5',
+                        'X-Object-Meta-key6': '6',
+                        'X-Object-Meta-key7': '7',
+                        'X-Object-Meta-key8': '8',
+                        'X-Object-Meta-key9': '9',
+                        'X-Object-Meta-key10': '10'}
+        self.storlet_file = 'source.txt'
+        self.container = 'myobjects'
+        self.dep_names = []
+        super(TestMetadataStorlet, self).setUp()
 
-    def test_metadata_get(self):
+    def test_metadata_get(self, params=None, global_params=None):
-        invoke_storlet(self.url, self.token, 'GET')
+        if params is not None:
+            querystring = ''
+            for key in params:
+                querystring += '%s=%s,' % (key, params[key])
+            querystring = querystring[:-1]
+        else:
+            querystring = None
+
+        headers = {'X-Run-Storlet': self.storlet_name}
+        original_headers, original_content = \
+            c.get_object(self.url, self.token,
+                         'myobjects', self.storlet_file,
+                         response_dict=dict(), headers=headers)
+        self.assertEqual(original_headers['X-Object-Meta-key1'.lower()], '1')
+        self.assertEqual(original_headers['X-Object-Meta-key2'.lower()], '2')
+        self.assertEqual(original_headers['X-Object-Meta-key3'.lower()], '3')
+        self.assertEqual(original_headers['X-Object-Meta-key4'.lower()], '4')
+        self.assertEqual(original_headers['X-Object-Meta-key5'.lower()], '5')
+        self.assertEqual(original_headers['X-Object-Meta-key6'.lower()], '6')
+        self.assertEqual(original_headers['X-Object-Meta-key7'.lower()], '7')
+        self.assertEqual(original_headers['X-Object-Meta-key8'.lower()], '8')
+        self.assertEqual(original_headers['X-Object-Meta-key9'.lower()], '9')
+        self.assertEqual(original_headers['X-Object-Meta-key10'.lower()], '10')
+        omv = original_headers['X-Object-Meta-override_key'.lower()]
+        self.assertEqual(omv, 'new_value')

tests/functional/test_test_storlet.py

@@ -13,20 +13,9 @@ See the License for the specific language governing permissions and
 Limitations under the License.
 -------------------------------------------------------------------------'''
 
-from __init__ import ACCOUNT
-from __init__ import AUTH_IP
-from __init__ import AUTH_PORT
-from __init__ import PASSWORD
-from __init__ import put_storlet_object
-from __init__ import USER_NAME
 import threading
-import unittest
-
 from swiftclient import client as c
-
+from __init__ import StorletFunctionalTest
-
-TEST_STORLET_NAME = 'test-10.jar'
-PATH_TO_STORLETS = '../../StorletSamples'
 
 
 class myTestThread (threading.Thread):
@@ -40,27 +29,26 @@ class myTestThread (threading.Thread):
         self.test_class.invokeTestStorlet("print", False)
 
 
-class TestTestStorlet(unittest.TestCase):
+class TestTestStorlet(StorletFunctionalTest):
     def setUp(self):
-        os_options = {'tenant_name': ACCOUNT}
+        self.storlet_dir = 'TestStorlet'
-        self.url, self.token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT
+        self.storlet_name = 'test-10.jar'
-                                          + "/v2.0", ACCOUNT + ":" + USER_NAME,
+        self.storlet_main = 'com.ibm.storlet.test.test1'
-                                          PASSWORD, os_options=os_options,
+        self.storlet_log = ''
-                                          auth_version='2.0')
+        self.headers = None
-        put_storlet_object(self.url,
+        self.storlet_file = ''
-                           self.token,
+        self.container = 'myobjects'
-                           TEST_STORLET_NAME,
+        self.dep_names = []
-                           "%s/TestStorlet/bin" % PATH_TO_STORLETS,
+        super(TestTestStorlet, self).setUp()
-                           '',
+
-                           'com.ibm.storlet.test.test1')
         c.put_object(self.url,
                      self.token,
-                     'myobjects',
+                     self.container,
                      'test_object',
                      'some content')
 
     def invokeTestStorlet(self, op, withlog=False):
-        headers = {'X-Run-Storlet': TEST_STORLET_NAME}
+        headers = {'X-Run-Storlet': self.storlet_name}
         if withlog is True:
             headers['X-Storlet-Generate-Log'] = 'True'
 
@@ -78,21 +66,22 @@ class TestTestStorlet(unittest.TestCase):
                                                 'storletlog', 'test.log',
                                                 None, None, None, None,
                                                 headers)
-                assert resp_headers.get('status') == 200
+                self.assertEqual(resp_headers.get('status'), 200)
                 gf.read()
-                assert resp_headers.get('status') == 200
+                self.assertEqual(resp_headers.get('status') == 200)
 
             if op == 'print':
-                assert get_response_status == 200
+                self.assertEqual(get_response_status, 200)
-                assert 'op' in get_text
+                self.assertTrue('op' in get_text)
-                assert 'print' in get_text
+                self.assertTrue('print' in get_text)
-                assert 'param2' in get_text
+                self.assertTrue('param2' in get_text)
-                assert 'val2' in get_text
+                self.assertTrue('val2' in get_text)
 
         except Exception:
             get_response_status = resp_dict.get('status')
             if op == 'crash':
-                assert get_response_status >= 500 or get_response_status == 404
+                self.assertTrue(get_response_status >= 500 or
+                                get_response_status == 404)
 
     def test_print(self):
         self.invokeTestStorlet("print", False)

tests/setup_functional_test.sh

@@ -61,3 +61,7 @@ ansible-playbook -s -i deploy/hosts host_storlet_engine.yml
 sudo chmod -R 777 /opt/ibm
 echo "Running create_default_tenant playbook"
 ansible-playbook -i deploy/hosts create_default_tenant.yml
+
+cd -
+cp cluster_config.json-sample cluster_config.json
+sudo chown $USER:$USER cluster_config.json