From 90187b0322f708dea4ca8cbda80264331ac58d6d Mon Sep 17 00:00:00 2001 From: Trevor McKay Date: Thu, 7 Aug 2014 11:31:28 -0400 Subject: [PATCH] Create etc/edp-examples directory Merge content from sahara-extra/edp-examples and integration/tests/resources into one directory under etc/edp-examples. This is part of the effort to ultimately move edp-examples out of the sahara-extra repo and eliminate duplication. The integration tests have been changed to reference the new etc/edp-examples directory, and an EDPJobInfo class has been added to eliminate path and config value duplication between the tests. Partial-implements: blueprint edp-move-examples Change-Id: I71b3cd21dcb9983fd6284a90316b12368481c700 --- etc/edp-examples/edp-java/README.rst | 54 ++++++++++++++++ .../edp-examples}/edp-java/edp-java.jar | Bin .../edp-java/oozie_command_line/README.rst | 22 +++++++ .../wordcount/job.properties | 23 +++++++ .../oozie_command_line/wordcount/workflow.xml | 49 +++++++++++++++ etc/edp-examples/edp-java/src/NOTICE.txt | 2 + .../edp-examples/edp-java/src}/WordCount.java | 0 .../edp-mapreduce}/edp-mapreduce.jar | Bin .../hadoop-mapreduce-examples-2.3.0.jar | Bin etc/edp-examples/pig-job/README.rst | 11 ++++ etc/edp-examples/pig-job/data/expected_output | 4 ++ etc/edp-examples/pig-job/data/input | 4 ++ .../edp-examples/pig-job/example.pig | 0 etc/edp-examples/pig-job/udf.jar | Bin 0 -> 1526 bytes sahara/tests/integration/tests/edp.py | 59 ++++++++++++++++++ .../tests/gating/test_cdh_gating.py | 43 ++----------- .../tests/gating/test_hdp2_gating.py | 37 +++-------- .../tests/gating/test_hdp_gating.py | 38 +++-------- .../tests/gating/test_vanilla_gating.py | 37 +++-------- .../tests/gating/test_vanilla_two_gating.py | 39 +++--------- .../tests/resources/edp-java/README | 4 -- .../integration/tests/resources/edp-lib.jar | Bin 3745 -> 0 bytes .../tests/vanilla_transient_cluster.py | 5 +- 23 files changed, 269 insertions(+), 162 deletions(-) create mode 100644 etc/edp-examples/edp-java/README.rst rename {sahara/tests/integration/tests/resources => etc/edp-examples}/edp-java/edp-java.jar (100%) create mode 100644 etc/edp-examples/edp-java/oozie_command_line/README.rst create mode 100644 etc/edp-examples/edp-java/oozie_command_line/wordcount/job.properties create mode 100644 etc/edp-examples/edp-java/oozie_command_line/wordcount/workflow.xml create mode 100644 etc/edp-examples/edp-java/src/NOTICE.txt rename {sahara/tests/integration/tests/resources/edp-java => etc/edp-examples/edp-java/src}/WordCount.java (100%) rename {sahara/tests/integration/tests/resources => etc/edp-examples/edp-mapreduce}/edp-mapreduce.jar (100%) rename {sahara/tests/integration/tests/resources => etc/edp-examples/hadoop2/edp-java}/hadoop-mapreduce-examples-2.3.0.jar (100%) create mode 100644 etc/edp-examples/pig-job/README.rst create mode 100644 etc/edp-examples/pig-job/data/expected_output create mode 100644 etc/edp-examples/pig-job/data/input rename sahara/tests/integration/tests/resources/edp-job.pig => etc/edp-examples/pig-job/example.pig (100%) create mode 100644 etc/edp-examples/pig-job/udf.jar delete mode 100644 sahara/tests/integration/tests/resources/edp-java/README delete mode 100644 sahara/tests/integration/tests/resources/edp-lib.jar diff --git a/etc/edp-examples/edp-java/README.rst b/etc/edp-examples/edp-java/README.rst new file mode 100644 index 00000000..60bb1e53 --- /dev/null +++ b/etc/edp-examples/edp-java/README.rst @@ -0,0 +1,54 @@ +===================== +EDP WordCount Example +===================== +Overview +======== + +``WordCount.java`` is a modified version of the WordCount example bundled with +version 1.2.1 of Apache Hadoop. It has been extended for use from a java action +in an Oozie workflow. The modification below allows any configuration values +from the ```` tag in an Oozie workflow to be set in the Configuration +object:: + + // This will add properties from the tag specified + // in the Oozie workflow. For java actions, Oozie writes the + // configuration values to a file pointed to by ooze.action.conf.xml + conf.addResource(new Path("file:///", + System.getProperty("oozie.action.conf.xml"))); + +In the example workflow, we use the ```` tag to specify user and +password configuration values for accessing swift objects. + +Compiling +========= + +To build the jar, add ``hadoop-core`` and ``commons-cli`` to the classpath. + +On a node running Ubuntu 13.04 with hadoop 1.2.1 the following commands +will compile ``WordCount.java`` from within the ``src`` directory:: + +$ mkdir wordcount_classes +$ javac -classpath /usr/share/hadoop/hadoop-core-1.2.1.jar:/usr/share/hadoop/lib/commons-cli-1.2.jar -d wordcount_classes WordCount.java +$ jar -cvf edp-java.jar -C wordcount_classes/ . + +Note, on a node with hadoop 2.3.0 the ``javac`` command above can be replaced with: + +$ javac -classpath /opt/hadoop-2.3.0/share/hadoop/common/hadoop-common-2.3.0.jar:/opt/hadoop-2.3.0/share/hadoop/mapreduce/hadoop-mapreduce-client-core-2.3.0.jar:/opt/hadoop-2.3.0/share/hadoop/common/lib/commons-cli-1.2.jar:/opt/hadoop-2.3.0/share/hadoop/mapreduce/lib/hadoop-annotations-2.3.0.jar -d wordcount_classes WordCount.java + +Running from the Sahara UI +=========================== + +Running the WordCount example from the Sahara UI is very similar to running a Pig, Hive, +or MapReduce job. + +1) Create a job binary that points to the ``edp-java.jar`` file +2) Create a ``Java`` job type and add the job binary to the ``libs`` value +3) Launch the job: + + + a) Add the input and output paths to ``args`` + b) If swift input or output paths are used, set the ``fs.swift.service.sahara.username`` and ``fs.swift.service.sahara.password`` + configuration values + c) The Sahara UI will prompt for the required ``main_class`` value and the optional ``java_opts`` value + + diff --git a/sahara/tests/integration/tests/resources/edp-java/edp-java.jar b/etc/edp-examples/edp-java/edp-java.jar similarity index 100% rename from sahara/tests/integration/tests/resources/edp-java/edp-java.jar rename to etc/edp-examples/edp-java/edp-java.jar diff --git a/etc/edp-examples/edp-java/oozie_command_line/README.rst b/etc/edp-examples/edp-java/oozie_command_line/README.rst new file mode 100644 index 00000000..5c9e991e --- /dev/null +++ b/etc/edp-examples/edp-java/oozie_command_line/README.rst @@ -0,0 +1,22 @@ +===================================================== +Running WordCount example from the Oozie command line +===================================================== + +1) Copy the *edp-java.jar* file from *sahara/edp-examples/edp-java* to *./wordcount/lib/edp-java.jar* + +2) Modify the *job.properties* file to specify the correct **jobTracker** and **nameNode** addresses for your cluster. + +3) Modify the *workflow.xml* file to contain the correct input and output paths. These paths may be Sahara swift urls or hdfs paths. + + * If swift urls are used, set the **fs.swift.service.sahara.username** and **fs.swift.service.sahara.password** + properties in the **** section. + +4) Upload the *wordcount* directory to hdfs + + $ hadoop fs -put wordcount wordcount + +5) Launch the job, specifying the correct oozie server and port + + $ oozie job -oozie http://oozie_server:port/oozie -config wordcount/job.properties -run + +6) Don't forget to create your swift input path! A Sahara swift url looks like *swift://container.sahara/object* diff --git a/etc/edp-examples/edp-java/oozie_command_line/wordcount/job.properties b/etc/edp-examples/edp-java/oozie_command_line/wordcount/job.properties new file mode 100644 index 00000000..5e9f4fb7 --- /dev/null +++ b/etc/edp-examples/edp-java/oozie_command_line/wordcount/job.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +nameNode=hdfs://1.2.3.4:8020 +jobTracker=1.2.3.4:8021 +queueName=default + +oozie.wf.application.path=${nameNode}/user/${user.name}/wordcount diff --git a/etc/edp-examples/edp-java/oozie_command_line/wordcount/workflow.xml b/etc/edp-examples/edp-java/oozie_command_line/wordcount/workflow.xml new file mode 100644 index 00000000..d8bc820f --- /dev/null +++ b/etc/edp-examples/edp-java/oozie_command_line/wordcount/workflow.xml @@ -0,0 +1,49 @@ + + + + + + ${jobTracker} + ${nameNode} + + + mapred.job.queue.name + ${queueName} + + + fs.swift.service.sahara.username + swiftuser + + + fs.swift.service.sahara.password + swiftpassword + + + org.openstack.sahara.examples.WordCount + swift://user.sahara/input + swift://user.sahara/output + + + + + + Java failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + diff --git a/etc/edp-examples/edp-java/src/NOTICE.txt b/etc/edp-examples/edp-java/src/NOTICE.txt new file mode 100644 index 00000000..62fc5816 --- /dev/null +++ b/etc/edp-examples/edp-java/src/NOTICE.txt @@ -0,0 +1,2 @@ +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). diff --git a/sahara/tests/integration/tests/resources/edp-java/WordCount.java b/etc/edp-examples/edp-java/src/WordCount.java similarity index 100% rename from sahara/tests/integration/tests/resources/edp-java/WordCount.java rename to etc/edp-examples/edp-java/src/WordCount.java diff --git a/sahara/tests/integration/tests/resources/edp-mapreduce.jar b/etc/edp-examples/edp-mapreduce/edp-mapreduce.jar similarity index 100% rename from sahara/tests/integration/tests/resources/edp-mapreduce.jar rename to etc/edp-examples/edp-mapreduce/edp-mapreduce.jar diff --git a/sahara/tests/integration/tests/resources/hadoop-mapreduce-examples-2.3.0.jar b/etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.3.0.jar similarity index 100% rename from sahara/tests/integration/tests/resources/hadoop-mapreduce-examples-2.3.0.jar rename to etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.3.0.jar diff --git a/etc/edp-examples/pig-job/README.rst b/etc/edp-examples/pig-job/README.rst new file mode 100644 index 00000000..2dfc5147 --- /dev/null +++ b/etc/edp-examples/pig-job/README.rst @@ -0,0 +1,11 @@ +Example Pig job +=============== + +This script trims spaces in input text + +This sample pig job is based on examples in Chapter 11 of +"Hadoop: The Definitive Guide" by Tom White, published by O'Reilly Media. +The original code can be found in a maven project at + +https://github.com/tomwhite/hadoop-book + diff --git a/etc/edp-examples/pig-job/data/expected_output b/etc/edp-examples/pig-job/data/expected_output new file mode 100644 index 00000000..60a01427 --- /dev/null +++ b/etc/edp-examples/pig-job/data/expected_output @@ -0,0 +1,4 @@ +pomegranate +banana +apple +lychee \ No newline at end of file diff --git a/etc/edp-examples/pig-job/data/input b/etc/edp-examples/pig-job/data/input new file mode 100644 index 00000000..d2079b57 --- /dev/null +++ b/etc/edp-examples/pig-job/data/input @@ -0,0 +1,4 @@ + pomegranate + banana + apple + lychee \ No newline at end of file diff --git a/sahara/tests/integration/tests/resources/edp-job.pig b/etc/edp-examples/pig-job/example.pig similarity index 100% rename from sahara/tests/integration/tests/resources/edp-job.pig rename to etc/edp-examples/pig-job/example.pig diff --git a/etc/edp-examples/pig-job/udf.jar b/etc/edp-examples/pig-job/udf.jar new file mode 100644 index 0000000000000000000000000000000000000000..970001c7ffa86ebb873eb2eff111f704d8ac59b7 GIT binary patch literal 1526 zcmWIWW@Zs#;Nak3*d52}%76qo8CV#6T|*poJ^kGD|D9rBU}gyLX6FE@V1guI?^rFjlMIV9(t zaeCzpo{xJ{XIrmzyZNPSPmKEMqkED+{F3{wWB6Xux$bP1(U~W_%NJg+I9hmT(_@<} zv8z{!+peCtO!wKoOoJCE_dE_QeOI(N?bKJU?>pmic9t6Nolt*3zHslkx2xhl*q&ZD zDfxz`y#a5$OWmej+PPa!Y5Y2yc*4kBZdRGAxMK9ar#F_xF17i>bJqBG+eh%*9F~a zFvg2|+-|3xt=G0iUw`d)bOzW>_T5C>zFZy(4`thkxg0Cf=E#4$vo)%mdERlEuRe2h zZmG_xcb*`}51_tZ2by8 zi=^>|r^-)BjCX9grP(8V-J*4d+G`!#;xA`{A7qO}1w<{;S@tsdyUnrk0|Dlzk;L5)Yp^w&<2K6XpFYs3!RJ1)*dG?f>g@IuXu#gMzW@Hj!KrIOV literal 0 HcmV?d00001 diff --git a/sahara/tests/integration/tests/edp.py b/sahara/tests/integration/tests/edp.py index 7a3c0fb2..72ffc32d 100644 --- a/sahara/tests/integration/tests/edp.py +++ b/sahara/tests/integration/tests/edp.py @@ -26,7 +26,66 @@ from sahara.tests.integration.tests import base from sahara.utils import edp +class EDPJobInfo(object): + PIG_PATH = 'etc/edp-examples/pig-job/' + JAVA_PATH = 'etc/edp-examples/edp-java/' + MAPREDUCE_PATH = 'etc/edp-examples/edp-mapreduce/' + HADOOP2_JAVA_PATH = 'etc/edp-examples/hadoop2/edp-java/' + + def read_pig_example_script(self): + return open(self.PIG_PATH + 'example.pig').read() + + def read_pig_example_jar(self): + return open(self.PIG_PATH + 'udf.jar').read() + + def read_java_example_lib(self, hadoop_vers=1): + if hadoop_vers == 1: + return open(self.JAVA_PATH + 'edp-java.jar').read() + return open(self.HADOOP2_JAVA_PATH + ( + 'hadoop-mapreduce-examples-2.3.0.jar')).read() + + def java_example_configs(self, hadoop_vers=1): + if hadoop_vers == 1: + return { + 'configs': { + 'edp.java.main_class': + 'org.openstack.sahara.examples.WordCount' + } + } + + return { + 'configs': { + 'edp.java.main_class': + 'org.apache.hadoop.examples.QuasiMonteCarlo' + }, + 'args': ['10', '10'] + } + + def read_mapreduce_example_jar(self): + return open(self.MAPREDUCE_PATH + 'edp-mapreduce.jar').read() + + def mapreduce_example_configs(self): + return { + 'configs': { + 'mapred.mapper.class': 'org.apache.oozie.example.SampleMapper', + 'mapred.reducer.class': + 'org.apache.oozie.example.SampleReducer' + } + } + + def mapreduce_streaming_configs(self): + return { + "configs": { + "edp.streaming.mapper": "/bin/cat", + "edp.streaming.reducer": "/usr/bin/wc" + } + } + + class EDPTest(base.ITestCase): + def setUp(self): + super(EDPTest, self).setUp() + self.edp_info = EDPJobInfo() def _create_data_source(self, name, data_type, url, description=''): return self.sahara.data_sources.create( diff --git a/sahara/tests/integration/tests/gating/test_cdh_gating.py b/sahara/tests/integration/tests/gating/test_cdh_gating.py index 592fed39..d6fe2523 100644 --- a/sahara/tests/integration/tests/gating/test_cdh_gating.py +++ b/sahara/tests/integration/tests/gating/test_cdh_gating.py @@ -24,7 +24,6 @@ from sahara.tests.integration.tests import map_reduce from sahara.tests.integration.tests import scaling from sahara.tests.integration.tests import swift from sahara.utils import edp as utils_edp -from sahara.utils import files as f class CDHGatingTest(cluster_configs.ClusterConfigTest, @@ -203,11 +202,9 @@ class CDHGatingTest(cluster_configs.ClusterConfigTest, self._edp_test() def _edp_test(self): - path = 'tests/integration/tests/resources/' - # check pig - pig_job = f.get_file_text(path + 'edp-job.pig') - pig_lib = f.get_file_text(path + 'edp-lib.jar') + pig_job = self.edp_info.read_pig_example_script() + pig_lib = self.edp_info.read_pig_example_jar() self.edp_testing(job_type=utils_edp.JOB_TYPE_PIG, job_data_list=[{'pig': pig_job}], lib_data_list=[{'jar': pig_lib}], @@ -215,14 +212,8 @@ class CDHGatingTest(cluster_configs.ClusterConfigTest, hdfs_local_output=True) # check mapreduce - mapreduce_jar = f.get_file_text(path + 'edp-mapreduce.jar') - mapreduce_configs = { - 'configs': { - 'mapred.mapper.class': 'org.apache.oozie.example.SampleMapper', - 'mapred.reducer.class': - 'org.apache.oozie.example.SampleReducer' - } - } + mapreduce_jar = self.edp_info.read_mapreduce_example_jar() + mapreduce_configs = self.edp_info.mapreduce_example_configs() self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE, job_data_list=[], lib_data_list=[{'jar': mapreduce_jar}], @@ -231,37 +222,13 @@ class CDHGatingTest(cluster_configs.ClusterConfigTest, hdfs_local_output=True) # check mapreduce streaming - mapreduce_streaming_configs = { - 'configs': { - 'edp.streaming.mapper': '/bin/cat', - 'edp.streaming.reducer': '/usr/bin/wc' - } - } self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE_STREAMING, job_data_list=[], lib_data_list=[], - configs=mapreduce_streaming_configs, + configs=self.edp_info.mapreduce_streaming_configs(), swift_binaries=False, hdfs_local_output=True) - # check java - """ - java_jar = f.get_file_text( - path + 'hadoop-mapreduce-examples-2.3.0.jar') - java_configs = { - 'configs': { - 'edp.java.main_class': - 'org.apache.hadoop.examples.QuasiMonteCarlo' - }, - 'args': ['10', '10'] - } - self.edp_testing(utils_edp.JOB_TYPE_JAVA, - job_data_list=[], - lib_data_list=[{'jar': java_jar}], - configs=java_configs, - swift_binaries=False, - hdfs_local_output=True)""" - @b.errormsg("Failure while cluster scaling: ") def _check_scaling(self): change_list = [ diff --git a/sahara/tests/integration/tests/gating/test_hdp2_gating.py b/sahara/tests/integration/tests/gating/test_hdp2_gating.py index a954ba78..7454fdcf 100644 --- a/sahara/tests/integration/tests/gating/test_hdp2_gating.py +++ b/sahara/tests/integration/tests/gating/test_hdp2_gating.py @@ -21,7 +21,6 @@ from sahara.tests.integration.tests import edp from sahara.tests.integration.tests import scaling from sahara.tests.integration.tests import swift from sahara.utils import edp as utils_edp -from sahara.utils import files as f class HDP2GatingTest(swift.SwiftTest, scaling.ScalingTest, @@ -128,11 +127,10 @@ class HDP2GatingTest(swift.SwiftTest, scaling.ScalingTest, self._edp_test() def _edp_test(self): - path = 'tests/integration/tests/resources/' - # check pig - pig_job = f.get_file_text(path + 'edp-job.pig') - pig_lib = f.get_file_text(path + 'edp-lib.jar') + pig_job = self.edp_info.read_pig_example_script() + pig_lib = self.edp_info.read_pig_example_jar() + self.edp_testing(job_type=utils_edp.JOB_TYPE_PIG, job_data_list=[{'pig': pig_job}], lib_data_list=[{'jar': pig_lib}], @@ -140,14 +138,8 @@ class HDP2GatingTest(swift.SwiftTest, scaling.ScalingTest, hdfs_local_output=True) # check mapreduce - mapreduce_jar = f.get_file_text(path + 'edp-mapreduce.jar') - mapreduce_configs = { - 'configs': { - 'mapred.mapper.class': 'org.apache.oozie.example.SampleMapper', - 'mapred.reducer.class': - 'org.apache.oozie.example.SampleReducer' - } - } + mapreduce_jar = self.edp_info.read_mapreduce_example_jar() + mapreduce_configs = self.edp_info.mapreduce_example_configs() self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE, job_data_list=[], lib_data_list=[{'jar': mapreduce_jar}], @@ -156,27 +148,14 @@ class HDP2GatingTest(swift.SwiftTest, scaling.ScalingTest, hdfs_local_output=True) # check mapreduce streaming - mapreduce_streaming_configs = { - 'configs': { - 'edp.streaming.mapper': '/bin/cat', - 'edp.streaming.reducer': '/usr/bin/wc' - } - } self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE_STREAMING, job_data_list=[], lib_data_list=[], - configs=mapreduce_streaming_configs) + configs=self.edp_info.mapreduce_streaming_configs()) # check java - java_jar = f.get_file_text( - path + 'hadoop-mapreduce-examples-2.3.0.jar') - java_configs = { - 'configs': { - 'edp.java.main_class': - 'org.apache.hadoop.examples.QuasiMonteCarlo' - }, - 'args': ['10', '10'] - } + java_jar = self.edp_info.read_java_example_lib(2) + java_configs = self.edp_info.java_example_configs(2) self.edp_testing(utils_edp.JOB_TYPE_JAVA, job_data_list=[], lib_data_list=[{'jar': java_jar}], diff --git a/sahara/tests/integration/tests/gating/test_hdp_gating.py b/sahara/tests/integration/tests/gating/test_hdp_gating.py index e3838d4f..12ecd4b3 100644 --- a/sahara/tests/integration/tests/gating/test_hdp_gating.py +++ b/sahara/tests/integration/tests/gating/test_hdp_gating.py @@ -157,35 +157,14 @@ class HDPGatingTest(cinder.CinderVolumeTest, edp.EDPTest, # ---------------------------------EDP TESTING--------------------------------- - path = 'sahara/tests/integration/tests/resources/' - pig_job_data = open(path + 'edp-job.pig').read() - pig_lib_data = open(path + 'edp-lib.jar').read() - mapreduce_jar_data = open(path + 'edp-mapreduce.jar').read() + pig_job_data = self.edp_info.read_pig_example_script() + pig_lib_data = self.edp_info.read_pig_example_jar() + + mapreduce_jar_data = self.edp_info.read_mapreduce_example_jar() # This is a modified version of WordCount that takes swift configs - java_lib_data = open(path + 'edp-java/edp-java.jar').read() - java_configs = { - "configs": { - "edp.java.main_class": - "org.openstack.sahara.examples.WordCount" - } - } + java_lib_data = self.edp_info.read_java_example_lib() - mapreduce_configs = { - "configs": { - "mapred.mapper.class": - "org.apache.oozie.example.SampleMapper", - "mapred.reducer.class": - "org.apache.oozie.example.SampleReducer" - } - } - mapreduce_streaming_configs = { - "configs": { - "edp.streaming.mapper": - "/bin/cat", - "edp.streaming.reducer": "/usr/bin/wc" - } - } try: self.edp_testing(job_type=utils_edp.JOB_TYPE_PIG, job_data_list=[{'pig': pig_job_data}], @@ -195,17 +174,18 @@ class HDPGatingTest(cinder.CinderVolumeTest, edp.EDPTest, self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE, job_data_list=[], lib_data_list=[{'jar': mapreduce_jar_data}], - configs=mapreduce_configs, + configs=self.edp_info.mapreduce_example_configs(), swift_binaries=True, hdfs_local_output=True) self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE_STREAMING, job_data_list=[], lib_data_list=[], - configs=mapreduce_streaming_configs) + configs=( + self.edp_info.mapreduce_streaming_configs())) self.edp_testing(job_type=utils_edp.JOB_TYPE_JAVA, job_data_list=[], lib_data_list=[{'jar': java_lib_data}], - configs=java_configs, + configs=self.edp_info.java_example_configs(), pass_input_output_args=True) except Exception as e: diff --git a/sahara/tests/integration/tests/gating/test_vanilla_gating.py b/sahara/tests/integration/tests/gating/test_vanilla_gating.py index 199e9998..5147686d 100644 --- a/sahara/tests/integration/tests/gating/test_vanilla_gating.py +++ b/sahara/tests/integration/tests/gating/test_vanilla_gating.py @@ -265,34 +265,12 @@ class VanillaGatingTest(cinder.CinderVolumeTest, # ---------------------------------EDP TESTING--------------------------------- - path = 'sahara/tests/integration/tests/resources/' - pig_job_data = open(path + 'edp-job.pig').read() - pig_lib_data = open(path + 'edp-lib.jar').read() - mapreduce_jar_data = open(path + 'edp-mapreduce.jar').read() + pig_job_data = self.edp_info.read_pig_example_script() + pig_lib_data = self.edp_info.read_pig_example_jar() + mapreduce_jar_data = self.edp_info.read_mapreduce_example_jar() # This is a modified version of WordCount that takes swift configs - java_lib_data = open(path + 'edp-java/edp-java.jar').read() - java_configs = { - "configs": { - "edp.java.main_class": - "org.openstack.sahara.examples.WordCount" - } - } - - mapreduce_configs = { - "configs": { - "mapred.mapper.class": - "org.apache.oozie.example.SampleMapper", - "mapred.reducer.class": - "org.apache.oozie.example.SampleReducer" - } - } - mapreduce_streaming_configs = { - "configs": { - "edp.streaming.mapper": "/bin/cat", - "edp.streaming.reducer": "/usr/bin/wc" - } - } + java_lib_data = self.edp_info.read_java_example_lib() try: self.edp_testing(job_type=utils_edp.JOB_TYPE_PIG, job_data_list=[{'pig': pig_job_data}], @@ -302,17 +280,18 @@ class VanillaGatingTest(cinder.CinderVolumeTest, self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE, job_data_list=[], lib_data_list=[{'jar': mapreduce_jar_data}], - configs=mapreduce_configs, + configs=self.edp_info.mapreduce_example_configs(), swift_binaries=True, hdfs_local_output=True) self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE_STREAMING, job_data_list=[], lib_data_list=[], - configs=mapreduce_streaming_configs) + configs=( + self.edp_info.mapreduce_streaming_configs())) self.edp_testing(job_type=utils_edp.JOB_TYPE_JAVA, job_data_list=[], lib_data_list=[{'jar': java_lib_data}], - configs=java_configs, + configs=self.edp_info.java_example_configs(), pass_input_output_args=True) except Exception as e: diff --git a/sahara/tests/integration/tests/gating/test_vanilla_two_gating.py b/sahara/tests/integration/tests/gating/test_vanilla_two_gating.py index 1b99ea95..27b7980d 100644 --- a/sahara/tests/integration/tests/gating/test_vanilla_two_gating.py +++ b/sahara/tests/integration/tests/gating/test_vanilla_two_gating.py @@ -24,10 +24,6 @@ from sahara.tests.integration.tests import map_reduce from sahara.tests.integration.tests import scaling from sahara.tests.integration.tests import swift from sahara.utils import edp as utils_edp -from sahara.utils import files as f - - -RESOURCES_PATH = 'tests/integration/tests/resources/' class VanillaTwoGatingTest(cluster_configs.ClusterConfigTest, @@ -199,8 +195,10 @@ class VanillaTwoGatingTest(cluster_configs.ClusterConfigTest, self._edp_java_test() def _edp_pig_test(self): - pig_job = f.get_file_text(RESOURCES_PATH + 'edp-job.pig') - pig_lib = f.get_file_text(RESOURCES_PATH + 'edp-lib.jar') + + pig_job = self.edp_info.read_pig_example_script() + pig_lib = self.edp_info.read_pig_example_jar() + self.edp_testing(job_type=utils_edp.JOB_TYPE_PIG, job_data_list=[{'pig': pig_job}], lib_data_list=[{'jar': pig_lib}], @@ -208,14 +206,8 @@ class VanillaTwoGatingTest(cluster_configs.ClusterConfigTest, hdfs_local_output=True) def _edp_mapreduce_test(self): - mapreduce_jar = f.get_file_text(RESOURCES_PATH + 'edp-mapreduce.jar') - mapreduce_configs = { - 'configs': { - 'mapred.mapper.class': 'org.apache.oozie.example.SampleMapper', - 'mapred.reducer.class': - 'org.apache.oozie.example.SampleReducer' - } - } + mapreduce_jar = self.edp_info.read_mapreduce_example_jar() + mapreduce_configs = self.edp_info.mapreduce_example_configs() self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE, job_data_list=[], lib_data_list=[{'jar': mapreduce_jar}], @@ -224,27 +216,14 @@ class VanillaTwoGatingTest(cluster_configs.ClusterConfigTest, hdfs_local_output=True) def _edp_mapreduce_streaming_test(self): - mapreduce_streaming_configs = { - 'configs': { - 'edp.streaming.mapper': '/bin/cat', - 'edp.streaming.reducer': '/usr/bin/wc' - } - } self.edp_testing(job_type=utils_edp.JOB_TYPE_MAPREDUCE_STREAMING, job_data_list=[], lib_data_list=[], - configs=mapreduce_streaming_configs) + configs=self.edp_info.mapreduce_streaming_configs()) def _edp_java_test(self): - java_jar = f.get_file_text( - RESOURCES_PATH + 'hadoop-mapreduce-examples-2.3.0.jar') - java_configs = { - 'configs': { - 'edp.java.main_class': - 'org.apache.hadoop.examples.QuasiMonteCarlo' - }, - 'args': ['10', '10'] - } + java_jar = self.edp_info.read_java_example_lib(2) + java_configs = self.edp_info.java_example_configs(2) self.edp_testing(utils_edp.JOB_TYPE_JAVA, job_data_list=[], lib_data_list=[{'jar': java_jar}], diff --git a/sahara/tests/integration/tests/resources/edp-java/README b/sahara/tests/integration/tests/resources/edp-java/README deleted file mode 100644 index 4fa99240..00000000 --- a/sahara/tests/integration/tests/resources/edp-java/README +++ /dev/null @@ -1,4 +0,0 @@ -Compiled against Hadoop 1.2.1 -$ mkdir wordcount_classes -$ javac -classpath /usr/share/hadoop/hadoop-core-1.2.1.jar:/usr/share/hadoop/lib/commons-cli-1.2.jar -d wordcount_classes WordCount.java -$ jar -cvf edp-java.jar -C wordcount_classes/ . diff --git a/sahara/tests/integration/tests/resources/edp-lib.jar b/sahara/tests/integration/tests/resources/edp-lib.jar deleted file mode 100644 index 2eba7023ec8e6cf6ec5a0df4f32a8c976f94e946..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3745 zcmb7G2{@GN7awIAMGdnsL_#COFiVuR#u{TE+9O+K9WvIGB64#HN%o6DiEO!+C`(G& zYDz*v$k?}pO8mdMWw|x?KKFm-`M&R+=RN25p7XxvocCu!+Q_~M#FYKPdo=$V{CIH# zUqfwkB1X?hN8aRz8W^MosM#G_I_(5ZUjiQ3dNo6$k)Dn=*<8+0=c8d$qX7vcN9QGB z&@D~R@0np+3B4cA%Idc~l_g;`z+3|v2V)R2)sjBtNnvJCk{OwrjJ^xowweDr!3^#X zZ8ZA`%;*z5^sD=PIke;Le^dHw{~Lr3JCPsb90@<=$2pC9QhB`cIB3N=5J)8z^|u(< z0q1NfF01bUsQ&& z&@*6h6l()L4_yky&eYS|*@5aSXX|Y3;gL)}VGJe0)Lyw(j2G=^BSOOb7w9h=0>#h@ zl8)88r6Cg8^wME3XMxz{EN8xNQeOfo5BoNUqDHQx9)7;^yLW6Bm|sCLzTA5v=^P`e zqO|8iCZ&z2C!KD5H+haffq%%PX7FlZ^CV~zPGTWBoSoRTM=x>DAP>fUBKAFp z7-fd>bgCUe@Fa88LmR`xqi^TWM3bT>0*lL!4S4D%ClZ_Ul@obr9LHP_TUf>x9gIHv z?v4^%rpni@8)7=8O=&l7Y5SitZj*CAcf!Y>^x)l;$S`-lMnf)#vb^)# z^eh{5$-bOsWg!kC z1XP}FTODtg-H0g@N%D!O zeolYbJ~h@e8SzE)Rl#s2q~^tkYr~EgJUP#YRqz(h#H7i88kj8yA7ba#rl12HOKl?< z0v5a>ORe%3Upt;qC-*0k1aKMw9MBN~PIB_C2iJ`dM|2eN54X<6CPDa%b%RnP-;~Twq+KhV0^&_wNiAKu5u(j{N5Ra|H2op)FEY>HD(B zQ>C&eQzejZiK_9TRUvr~E+NEjs*BC-t+=)z<(DN)J1j14yNi7;vK!)ik*M4WrW-M~ zXi&v9OB=P0(o=RcFtFk3NtwAkuZQbFe3m*%eL*mZK10(6m1~l}35#1v2$Q;YqzNQw zFrK3QsjBz5Z0B1QKZ%GRh0_GpEu-2FL@IrOuGHE0Ca@+!UInkT9t=n`e z@2X%dd;p$%+hi&w&;rigb_bm6N{}|AMw-}+hAfPt%H8`@cPd?@?25(&yE+F`r036DH}&A&^4zn3 z%=G|10cL)u6oxN1-#$EeU&Sh&lRTawr~mqy-Al23Huc3I>yi*gE? z9dbAoS+6!9+SnC2(SY5HEbrSSQKcWe9J9j|m#8?eB8u!0WSdd_=n>N&H}!OaKNwkp zR(&5!t;8QCB6aTW6?qQPCZpma_t=<@IK+Dl4sU)J|BTfIzUMJq316pC438xzWq&7Mvn7_gI zKvTN2Xoxp5HFX94>OrI0AEEKGM&+`J{=G^(m|l`dj`DqM+cT>YDs*8JffCRM>xVhvz3K)kgbA9 z9nhB}5sbxM6)5>Xa%T$&j>N05NUhhRQv%>;CmavdrA9tOQTW;oykOPmkwe`FUM10@ z*;;F*^PUeArUO1c zM8V`T4H^R7i_W7r4K#~m92N=bv${rS1NJwHh}TX*pR`|)^9w!0F1$IQ;a%-ysK`m@ z02)_j$&1Ge6uT60?|gEcQ<-R!b$es+29!G+IRj_-o{)hH+5^tNj5i)SJ#=NvzClXk zCA;Ngr{ZYe%TojTes)x$!83hUSaiR-TX*j;W|AzRk1!IuPb`T( zqxL{YJ@$*8+$&kN#|#gyva$e2>Dt57{m1)ft7=Z>(H~AG`l!TF-ai5nPI&j-4XxX^ z(hxu(`IB=RtJfqjU|y@M+^V@8f0i<=O8vVbD-bXU z18@Y_Hn%N`Y(U=v`j^~)2XUjg$oaT92S*brwPK*)$!~DO$>0$$EGMo)_E`EWt~Z5u-ypf<)~Dh1=ccAXh83q0C(8;op6TI07EvC0aA;8qY;u%w*4)57q}a44 z>3&y5W-?5eF)LUbGFHZ}1-<`%f4q>KfX4zDT}6tWm(Y5W46i@@wj}jD_>#2+Z!g2Y zWJYCo2JEG>xb(j8W4ID)&#YbUY#%44KVx)HAAvRYZoFXG>n7+3>wkN#oqbCVIK|<} z>}#EGLx)0qZ`ztuBcw?EBi?40}|igDxXMw{@H`8r$s6G(&G zwcSRM{L=#4bCjb$hs}9+L$YFw7R;WUL1*I^qjB%IU9{c=Hnv_Oyn#@x9ZjvQ4U~dL zg?f;-96Dcbsn}Ky7{s)XTQ)&d{k;~sR+0q_QUy)`RxQnN26o;DcohD>6L33A0g{92 zVDCZpGHxbe^kwx7O%L2LpE8ssHTRo&$uyXQjW8xyEE$V0z+;Q?3KnkjymUouAs!3b zgcS0)k%@qx9%OU~jUb0OzbbLEpIf-%mMQSzCc-DHJ?kJW17SayHFyK&u@iB_E^8<1 zW-9kafMl|5fNlI|stm*c2m~}vWPVq3f6a_Fb!I9vCF9cHl6f7GUPJhmy)yajcVRAE z2MoWn-8EN#X2i_G{4Pw3L16e3p