diff --git a/etc/scenario/sahara-ci/ambari-2.3.yaml.mako b/etc/scenario/sahara-ci/ambari-2.3.yaml.mako index db67b704..a28c1491 100644 --- a/etc/scenario/sahara-ci/ambari-2.3.yaml.mako +++ b/etc/scenario/sahara-ci/ambari-2.3.yaml.mako @@ -42,4 +42,8 @@ clusters: name: ${cluster_name} scenario: - run_jobs - edp_jobs_flow: hadoop_2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job diff --git a/etc/scenario/sahara-ci/cdh-5.3.0.yaml.mako b/etc/scenario/sahara-ci/cdh-5.3.0.yaml.mako index 5b8a9b04..9a011262 100644 --- a/etc/scenario/sahara-ci/cdh-5.3.0.yaml.mako +++ b/etc/scenario/sahara-ci/cdh-5.3.0.yaml.mako @@ -68,4 +68,8 @@ clusters: scenario: - run_jobs - sentry - edp_jobs_flow: hadoop_2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job \ No newline at end of file diff --git a/etc/scenario/sahara-ci/cdh-5.4.0.yaml.mako b/etc/scenario/sahara-ci/cdh-5.4.0.yaml.mako index 0406d7cb..230676e6 100644 --- a/etc/scenario/sahara-ci/cdh-5.4.0.yaml.mako +++ b/etc/scenario/sahara-ci/cdh-5.4.0.yaml.mako @@ -73,4 +73,9 @@ clusters: scenario: - run_jobs - sentry - edp_jobs_flow: cdh_flow + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job + - spark_wordcount diff --git a/etc/scenario/sahara-ci/edp.yaml.mako b/etc/scenario/sahara-ci/edp.yaml.mako index f01cd720..1a570aa4 100644 --- a/etc/scenario/sahara-ci/edp.yaml.mako +++ b/etc/scenario/sahara-ci/edp.yaml.mako @@ -1,9 +1,8 @@ edp_jobs_flow: - hadoop_2: + pig_job: - type: Pig input_datasource: - type: hdfs - hdfs_username: hadoop + type: swift source: etc/edp-examples/edp-pig/trim-spaces/data/input output_datasource: type: hdfs @@ -14,6 +13,7 @@ edp_jobs_flow: additional_libs: - type: swift source: etc/edp-examples/edp-pig/trim-spaces/udf.jar + mapreduce_job: - type: MapReduce input_datasource: type: swift @@ -27,6 +27,7 @@ edp_jobs_flow: configs: mapred.mapper.class: org.apache.oozie.example.SampleMapper mapred.reducer.class: org.apache.oozie.example.SampleReducer + mapreduce_streaming_job: - type: MapReduce.Streaming input_datasource: type: swift @@ -37,6 +38,7 @@ edp_jobs_flow: configs: edp.streaming.mapper: /bin/cat edp.streaming.reducer: /usr/bin/wc + java_job: - type: Java additional_libs: - type: database @@ -46,7 +48,7 @@ edp_jobs_flow: args: - 10 - 10 - spark_edp: + spark_pi: - type: Spark main_lib: type: database @@ -55,6 +57,7 @@ edp_jobs_flow: edp.java.main_class: org.apache.spark.examples.SparkPi args: - 4 + spark_wordcount: - type: Spark input_datasource: type: swift @@ -69,20 +72,6 @@ edp_jobs_flow: fs.swift.service.sahara.password: ${OS_PASSWORD} args: - '{input_datasource}' - transient: - - type: Pig - input_datasource: - type: swift - source: etc/edp-examples/edp-pig/trim-spaces/data/input - output_datasource: - type: hdfs - destination: /user/hadoop/edp-output - main_lib: - type: swift - source: etc/edp-examples/edp-pig/trim-spaces/example.pig - additional_libs: - - type: swift - source: etc/edp-examples/edp-pig/trim-spaces/udf.jar mapr: - type: Pig input_datasource: @@ -129,63 +118,3 @@ edp_jobs_flow: args: - 10 - 10 - cdh_flow: - - type: Pig - input_datasource: - type: swift - source: etc/edp-examples/edp-pig/trim-spaces/data/input - output_datasource: - type: hdfs - destination: /user/hadoop/edp-output - main_lib: - type: swift - source: etc/edp-examples/edp-pig/trim-spaces/example.pig - additional_libs: - - type: swift - source: etc/edp-examples/edp-pig/trim-spaces/udf.jar - - type: MapReduce - input_datasource: - type: swift - source: etc/edp-examples/edp-pig/trim-spaces/data/input - output_datasource: - type: hdfs - destination: /user/hadoop/edp-output - additional_libs: - - type: database - source: etc/edp-examples/edp-mapreduce/edp-mapreduce.jar - configs: - mapred.mapper.class: org.apache.oozie.example.SampleMapper - mapred.reducer.class: org.apache.oozie.example.SampleReducer - - type: MapReduce.Streaming - input_datasource: - type: swift - source: etc/edp-examples/edp-pig/trim-spaces/data/input - output_datasource: - type: hdfs - destination: /user/hadoop/edp-output - configs: - edp.streaming.mapper: /bin/cat - edp.streaming.reducer: /usr/bin/wc - - type: Java - additional_libs: - - type: database - source: etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.6.0.jar - configs: - edp.java.main_class: org.apache.hadoop.examples.QuasiMonteCarlo - args: - - 10 - - 10 - - type: Spark - input_datasource: - type: swift - source: etc/edp-examples/edp-spark/sample_input.txt - main_lib: - type: database - source: etc/edp-examples/edp-spark/spark-wordcount.jar - configs: - edp.java.main_class: sahara.edp.spark.SparkWordCount - edp.spark.adapt_for_swift: true - fs.swift.service.sahara.username: ${OS_USERNAME} - fs.swift.service.sahara.password: ${OS_PASSWORD} - args: - - '{input_datasource}' diff --git a/etc/scenario/sahara-ci/fake.yaml.mako b/etc/scenario/sahara-ci/fake.yaml.mako index aada1c46..31b6e143 100644 --- a/etc/scenario/sahara-ci/fake.yaml.mako +++ b/etc/scenario/sahara-ci/fake.yaml.mako @@ -29,4 +29,4 @@ clusters: - operation: add node_group: worker size: 1 - edp_jobs_flow: transient + edp_jobs_flow: pig_job diff --git a/etc/scenario/sahara-ci/hdp-2.0.6.yaml.mako b/etc/scenario/sahara-ci/hdp-2.0.6.yaml.mako index 054fe414..8629054a 100644 --- a/etc/scenario/sahara-ci/hdp-2.0.6.yaml.mako +++ b/etc/scenario/sahara-ci/hdp-2.0.6.yaml.mako @@ -45,4 +45,8 @@ clusters: - operation: add node_group: worker size: 1 - edp_jobs_flow: hadoop_2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job diff --git a/etc/scenario/sahara-ci/spark-1.0.0.yaml.mako b/etc/scenario/sahara-ci/spark-1.0.0.yaml.mako index ecdef183..cac2d028 100644 --- a/etc/scenario/sahara-ci/spark-1.0.0.yaml.mako +++ b/etc/scenario/sahara-ci/spark-1.0.0.yaml.mako @@ -29,4 +29,6 @@ clusters: - operation: add node_group: worker size: 1 - edp_jobs_flow: spark_edp + edp_jobs_flow: + - spark_pi + - spark_wordcount diff --git a/etc/scenario/sahara-ci/spark-1.3.1.yaml.mako b/etc/scenario/sahara-ci/spark-1.3.1.yaml.mako index 21cd6fc8..375af5a3 100644 --- a/etc/scenario/sahara-ci/spark-1.3.1.yaml.mako +++ b/etc/scenario/sahara-ci/spark-1.3.1.yaml.mako @@ -30,4 +30,6 @@ clusters: - operation: add node_group: worker size: 1 - edp_jobs_flow: spark_edp + edp_jobs_flow: + - spark_pi + - spark_wordcount \ No newline at end of file diff --git a/etc/scenario/sahara-ci/transient.yaml.mako b/etc/scenario/sahara-ci/transient.yaml.mako index a5a86899..a8268f62 100644 --- a/etc/scenario/sahara-ci/transient.yaml.mako +++ b/etc/scenario/sahara-ci/transient.yaml.mako @@ -48,4 +48,5 @@ clusters: scenario: - run_jobs - transient - edp_jobs_flow: transient + edp_jobs_flow: pig_job + diff --git a/etc/scenario/sahara-ci/vanilla-2.6.0.yaml.mako b/etc/scenario/sahara-ci/vanilla-2.6.0.yaml.mako index 11096cc4..18a603e0 100644 --- a/etc/scenario/sahara-ci/vanilla-2.6.0.yaml.mako +++ b/etc/scenario/sahara-ci/vanilla-2.6.0.yaml.mako @@ -83,4 +83,8 @@ clusters: - operation: add node_group: worker-nm size: 1 - edp_jobs_flow: hadoop_2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job diff --git a/etc/scenario/sahara-ci/vanilla-2.7.1.yaml.mako b/etc/scenario/sahara-ci/vanilla-2.7.1.yaml.mako index 73863374..615d71ea 100644 --- a/etc/scenario/sahara-ci/vanilla-2.7.1.yaml.mako +++ b/etc/scenario/sahara-ci/vanilla-2.7.1.yaml.mako @@ -69,4 +69,8 @@ clusters: - operation: add node_group: worker-nm size: 2 - edp_jobs_flow: hadoop_2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job