Cleanup deploy_storlet and the tests

Related the patch https://review.openstack.org/#/c/412766

This patch addresses:

- Cleanup stdin, stdout sequence which can be maintained by just
  dict map rather than if/elif/else for each supported language
  at main() in storlet/tools/deploy_storlet.py

- Add basic unit tests for deploy_storlet (no error test cases exist with this)

- Make mixin class at tests/functional/common/mixins.py to avoid
  redandant copy/paste code for java/python functional tests

Change-Id: I6098954a0abcc57314b6f4d0af3efa5d373ad37c

storlets/tools/cluster_config_parser.py

@@ -30,7 +30,8 @@ import json
 class ClusterConfig(object):
 
     def __init__(self, config_path):
-        conf_string = open(config_path, 'r').read()
+        with open(config_path, 'r') as f:
+            conf_string = f.read()
         self.conf = json.loads(conf_string)
         self._auth_version = '3'
 

storlets/tools/deploy_storlet.py

@@ -22,7 +22,43 @@ from storlets.tools.utils import get_auth, deploy_storlet
 CLS_SUFFIX = '.class'
 
 
+class StdinReader(object):
+    """
+    Helper reader class to get stdin and map the value to specified key
+
+    :param key: a string but it's never used in this class just a reference
+                for the value should be assigned
+    :param run_once: bool. If set False, continue to readline until blank line
+                     in __call__ method.
+    :param callback: function which should be called with the value after
+                     all lines were read.
+    """
+    def __init__(self, key, run_once=True, callback=None):
+        self.key = key
+        self.run_once = run_once
+        self.callback = callback
+
+    def __call__(self):
+        """
+        :return: string when run_once == True else list
+        """
+        line = sys.stdin.readline().rstrip()
+        if not self.run_once:
+            value = []
+            while line:
+                value.append(line)
+                line = sys.stdin.readline().rstrip()
+        else:
+            value = line
+
+        if self.callback:
+            self.callback(value)
+
+        return value
+
+
 def list_classes(storlet_file):
+    print("Your jar file contains the following classes:")
     with zipfile.ZipFile(storlet_file, 'r') as zfile:
         for f in zfile.infolist():
             name = f.filename
@@ -31,6 +67,25 @@ def list_classes(storlet_file):
                       name[:-len(CLS_SUFFIX)].replace('/', '.'))
 
 
+MESSAGES = {
+    'Java': iter([
+        ("Enter absolute path to storlet jar file: ",
+         StdinReader("storlet", callback=list_classes)),
+        ("Please enter fully qualified storlet main class "
+         "(choose from the list above): ", StdinReader("storlet_main_class")),
+        ("Please enter dependency files "
+         "(leave a blank line when you are done):",
+         StdinReader("dependencies", False))]),
+    'Python': iter([
+        ("Enter absolute path to storlet file: ", StdinReader("storlet")),
+        ("Please enter fully qualified storlet main class "
+         "<filename.ClassName>: ", StdinReader("storlet_main_class")),
+        ("Please enter dependency files "
+         "(leave a blank line when you are done): ",
+         StdinReader("dependencies", False))]),
+}
+
+
 def usage():
     print("Useage: deploy_storlet.py <path to conf>")
 
@@ -38,38 +93,24 @@ def usage():
 def main(argv):
     if len(argv) != 1:
         usage()
-        return -1
+        return 1
     conf = ClusterConfig(argv[0])
     url, token = get_auth(conf, conf.admin_user, conf.admin_password)
-    sys.stdout.write("Enter storlet language (java or python): ")
-    storlet_language = sys.stdin.readline().rstrip()
+    print("Enter storlet language (java or python): ")
+    storlet_language = sys.stdin.readline().rstrip().title()
 
-    if storlet_language.lower() == 'java':
-        sys.stdout.write("Enter absolute path to storlet jar file: ")
-        storlet_file = sys.stdin.readline().rstrip()
-        print("Your jar file contains the following classes:")
-        list_classes(storlet_file)
-        sys.stdout.write("Please enter fully qualified storlet main class " +
-                         "(choose from the list above): ")
-        storlet_main_class = sys.stdin.readline().rstrip()
-    elif storlet_language.lower() == 'python':
-        sys.stdout.write("Enter absolute path to storlet file: ")
-        storlet_file = sys.stdin.readline().rstrip()
-        sys.stdout.write("Please enter fully qualified storlet main class: ")
-        storlet_main_class = sys.stdin.readline().rstrip()
-    else:
-        print("unsupported storlet_language.")
-        return 0
+    if storlet_language not in MESSAGES:
+        print("The language you specified is not supported")
+        return 1
+
+    message_iter = MESSAGES[storlet_language]
+    options_dict = dict(language=storlet_language)
+    for message, reader in message_iter:
+        print(message)
+        options_dict[reader.key] = reader()
+
+    deploy_storlet(url, token, **options_dict)
 
-    print("Please enter dependency files (leave a blank line when you are "
-          "done):")
-    dependency_files = []
-    dependency_file = sys.stdin.readline().rstrip()
-    while dependency_file:
-        dependency_files.append(dependency_file)
-        dependency_file = sys.stdin.readline().rstrip()
-    deploy_storlet(url, token, storlet_file, storlet_main_class,
-                   dependency_files, storlet_language)
     print("Storlet deployment complete")
     return 0
 

tests/functional/common/mixins.py

@@ -0,0 +1,68 @@
+# Copyright (c) 2016 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pexpect
+import swiftclient
+import os
+import hashlib
+from storlets.tools.cluster_config_parser import ClusterConfig
+from storlets.tools.utils import get_auth
+
+EXPECT = 'expect'
+SEND_LINE = 'sendline'
+
+
+class DeployTestMixin(object):
+    def assertUploadedFile(self, container, file_path):
+        # Make sure the existence in the swift inside
+        conf = ClusterConfig(self.conf_file)
+        url, token = get_auth(conf, conf.admin_user, conf.admin_password)
+
+        expected_file = os.path.basename(file_path)
+        resp_headers = swiftclient.client.head_object(
+            url, token, container, expected_file)
+        hasher = hashlib.md5()
+        with open(file_path) as f:
+            hasher.update(f.read())
+        self.assertEqual(resp_headers['etag'], hasher.hexdigest())
+
+    def test_deploy_storlet(self):
+        child = pexpect.spawn('python %s %s' % (self.deploy_storlet_path,
+                                                self.conf_file))
+        try:
+            for message, command in self.scenario:
+                # FIXME(kota_): i don't get yet why the *expect* call stack
+                #               stdout from print function but it seems safe
+                #               to assert the output by hand rather than using
+                #               expect method
+                if command == EXPECT:
+                    line = child.readline()
+                    self.assertEqual(message.strip(), line.strip())
+                elif command == SEND_LINE:
+                    child.sendline(message)
+                    line = child.readline()
+                    self.assertEqual(message.strip(), line.strip())
+                else:
+                    self.fail("Unexpected scenario found %s, %s"
+                              % (message, command))
+        except pexpect.EOF as err:
+            self.fail(
+                'Expected message "%s" not found: %s' % (message, err))
+
+        # Make sure the existence in the swift inside
+        # for storlet app
+        self.assertUploadedFile('storlet', self.execdep_storlet_file)
+        # for dependency
+        self.assertUploadedFile('dependency', self.execdep_storlet_dep_file)

tests/functional/java/test_deploy_storlet.py

@@ -15,15 +15,14 @@
 # limitations under the License.
 
 import os
-import pexpect
-from tests.functional import StorletBaseFunctionalTest, PATH_TO_STORLETS, \
-    CONSOLE_TIMEOUT
-from tests.functional.java import BIN_DIR
 import unittest
 from storlets.tools import deploy_storlet
+from tests.functional import StorletBaseFunctionalTest, PATH_TO_STORLETS
+from tests.functional.java import BIN_DIR
+from tests.functional.common.mixins import DeployTestMixin, EXPECT, SEND_LINE
 
 
-class TestDeployStorlet(StorletBaseFunctionalTest):
+class TestDeployStorlet(DeployTestMixin, StorletBaseFunctionalTest):
     def setUp(self):
         super(TestDeployStorlet, self).setUp()
         self.deploy_storlet_path = os.path.abspath(deploy_storlet.__file__)
@@ -31,31 +30,27 @@ class TestDeployStorlet(StorletBaseFunctionalTest):
                                                  'java',
                                                  'ExecDepStorlet',
                                                  BIN_DIR)
-        self.execdep_storlet_jar_file = os.path.join(self.execdep_storlet_path,
-                                                     'execdepstorlet-1.0.jar')
+        self.execdep_storlet_file = os.path.join(self.execdep_storlet_path,
+                                                 'execdepstorlet-1.0.jar')
         self.execdep_storlet_dep_file = os.path.join(self.execdep_storlet_path,
                                                      'get42')
 
-        self.timeout = CONSOLE_TIMEOUT
-
-    def test_deploy_storlet_util_java(self):
-        child = pexpect.spawn('python %s %s' % (self.deploy_storlet_path,
-                                                self.conf_file))
-        child.expect('Enter storlet language.*',
-                     timeout=self.timeout)
-        child.sendline('java')
-        child.expect('Enter absolute path to storlet jar file.*:',
-                     timeout=self.timeout)
-        child.sendline(self.execdep_storlet_jar_file)
-        child.expect('org.openstack.storlet.execdep.ExecDepStorlet',
-                     timeout=self.timeout)
-        child.expect('Please enter fully qualified storlet main class.*',
-                     timeout=self.timeout)
-        child.sendline('org.openstack.storlet.execdep.ExecDepStorlet')
-        child.expect('Please enter dependency.*', timeout=self.timeout)
-        child.sendline(self.execdep_storlet_dep_file)
-        child.sendline('\n')
-        child.expect('Storlet deployment complete.*', timeout=self.timeout)
+        self.scenario = [
+            ('Enter storlet language (java or python): ', EXPECT),
+            ('java', SEND_LINE),
+            ('Enter absolute path to storlet jar file: ', EXPECT),
+            (self.execdep_storlet_file, SEND_LINE),
+            ('Your jar file contains the following classes:', EXPECT),
+            ('\t* org.openstack.storlet.execdep.ExecDepStorlet', EXPECT),
+            ('Please enter fully qualified storlet main class '
+             '(choose from the list above): ', EXPECT),
+            ('org.openstack.storlet.execdep.ExecDepStorlet', SEND_LINE),
+            ('Please enter dependency files '
+             '(leave a blank line when you are done): ', EXPECT),
+            (self.execdep_storlet_dep_file, SEND_LINE),
+            ('', SEND_LINE),  # DO NOT send \n but just empty due to sendline
+            ('Storlet deployment complete', EXPECT),
+        ]
 
 
 if __name__ == '__main__':

tests/functional/python/test_deploy_storlet.py

@@ -14,14 +14,13 @@
 # limitations under the License.
 
 import os
-import pexpect
-from tests.functional import StorletBaseFunctionalTest, PATH_TO_STORLETS, \
-    CONSOLE_TIMEOUT
 import unittest
 from storlets.tools import deploy_storlet
+from tests.functional import StorletBaseFunctionalTest, PATH_TO_STORLETS
+from tests.functional.common.mixins import DeployTestMixin, EXPECT, SEND_LINE
 
 
-class TestDeployStorlet(StorletBaseFunctionalTest):
+class TestDeployStorlet(DeployTestMixin, StorletBaseFunctionalTest):
     def setUp(self):
         super(TestDeployStorlet, self).setUp()
         self.deploy_storlet_path = os.path.abspath(deploy_storlet.__file__)
@@ -34,24 +33,20 @@ class TestDeployStorlet(StorletBaseFunctionalTest):
         self.execdep_storlet_dep_file = os.path.join(self.execdep_storlet_path,
                                                      'get42.sh')
 
-        self.timeout = CONSOLE_TIMEOUT
-
-    def test_deploy_storlet_util_python(self):
-        child = pexpect.spawn('python %s %s' % (self.deploy_storlet_path,
-                                                self.conf_file))
-        child.expect('Enter storlet language.*',
-                     timeout=self.timeout)
-        child.sendline('python')
-        child.expect('Enter absolute path to storlet file.*:',
-                     timeout=self.timeout)
-        child.sendline(self.execdep_storlet_file)
-        child.expect('Please enter fully qualified storlet main class.*',
-                     timeout=self.timeout)
-        child.sendline('exec_dep.ExecDepStorlet')
-        child.expect('Please enter dependency.*', timeout=self.timeout)
-        child.sendline(self.execdep_storlet_dep_file)
-        child.sendline('\n')
-        child.expect('Storlet deployment complete.*', timeout=self.timeout)
+        self.scenario = [
+            ('Enter storlet language (java or python): ', EXPECT),
+            ('python', SEND_LINE),
+            ('Enter absolute path to storlet file: ', EXPECT),
+            (self.execdep_storlet_file, SEND_LINE),
+            ('Please enter fully qualified storlet main class '
+             '<filename.ClassName>: ', EXPECT),
+            ('exec_dep.ExecDepStorlet', SEND_LINE),
+            ('Please enter dependency files '
+             '(leave a blank line when you are done): ', EXPECT),
+            (self.execdep_storlet_dep_file, SEND_LINE),
+            ('', SEND_LINE),  # DO NOT send \n but just empty due to sendline
+            ('Storlet deployment complete', EXPECT),
+        ]
 
 
 if __name__ == '__main__':

tests/unit/tools/extensions/test_ipython.py

@@ -138,7 +138,7 @@ class TestStorletMagics(unittest.TestCase):
 
                 self.assertEqual(
                     "You need to set OS_AUTH_URL, OS_USERNAME, OS_PASSWORD "
-                    "OS_PROJECT_NAME for Swift authentication",
+                    "and OS_PROJECT_NAME for Swift authentication",
                     e.exception.message)
 
     def test_storlet_auth_v2_not_supported(self):

tests/unit/tools/test_deploy_storlet.py

@@ -0,0 +1,76 @@
+# Copyright (c) 2016 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import mock
+from storlets.tools.deploy_storlet import main
+
+
+class TestDeployStorlet(unittest.TestCase):
+    # TODO(kota_): missing test cases like:
+    #              - no class found in the jar
+    #              - not a class in the list
+    #              - fail to open the jar file
+    #                (e.g. no such a file or directry)
+    def test_deploy_storlet_main_java(self):
+        class MockZipFile(mock.MagicMock):
+            def infolist(self):
+                mock_file = mock.MagicMock()
+                mock_file.filename = 'main_class.class'
+                return [mock_file]
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *args, **kwargs):
+                pass
+
+        with mock.patch('zipfile.ZipFile', MockZipFile):
+            self._test_deploy_storlet_main('java')
+
+    def test_deploy_storlet_main_python(self):
+        self._test_deploy_storlet_main('python')
+
+    def _test_deploy_storlet_main(self, lang):
+        # TODO(kota_): avoid mock for ClusterConfig and get_auth
+        get_auth_func_path = 'storlets.tools.deploy_storlet.get_auth'
+        deploy_storlet_path = 'storlets.tools.deploy_storlet.deploy_storlet'
+        with mock.patch('storlets.tools.deploy_storlet.ClusterConfig'), \
+                mock.patch(deploy_storlet_path) as mock_deploy_storlet, \
+                mock.patch(get_auth_func_path) as mock_auth:
+
+            mock_auth.return_value = ('url', 'token')
+            stdins = [
+                lang, 'storlet_file_path', 'main_class', ''
+            ]
+
+            class MockStdin(object):
+                def readline(self):
+                    return stdins.pop(0)
+
+            with mock.patch('sys.stdin', MockStdin()):
+                main(['dummy_config_path'])
+
+        # sanity
+        self.assertEqual(1, mock_deploy_storlet.call_count)
+        self.assertEqual(
+            mock.call('url', 'token', storlet='storlet_file_path',
+                      storlet_main_class='main_class', dependencies=[],
+                      language=lang.title()),
+            mock_deploy_storlet.call_args)
+
+
+if __name__ == '__main__':
+    unittest.main()