Put in Sahara repo actual scenario files

Now scenario files placed in Sahara repo aren't equal to scenario files
which actually is using in Sahara CI.

Also we want to use scenario files from Sahara repo, instead
Sahara CI repo, so we need to update this files.

Change-Id: Id6adb8f89934ffe4ceb85cf7c12359dca8eff09e

etc/scenario/sahara-ci/README.rst

@@ -5,3 +5,25 @@ Sahara CI
 The files in this directory are needed for the sahara continuous
 integration tests. Modifying these files will change the behavior of the
 tests.
+
+Details
+-------
+
+Key values:
+
+* %OS_USERNAME%, %OS_PASSWORD%, %OS_TENANT_NAME%, %OPENSTACK_HOST% - OpenStack credentials;
+* %NETWORK% - network type (neutron or nova-network);
+* %cluster_name% - name of cluster, which generating from $HOST-$ZUUL_CHANGE-$CLUSTER_HASH. Where:
+    * $HOST - host id (c1 - with neutron, c2 - with nova-network);
+    * $ZUUL_CHANGE - change number;
+    * $CLUSTER_HASH - hash, which generating for each cluster by using "uuid" python module;
+* %{plugin}_image% - name of image for each plugin;
+* flavor ids:
+    * %ci_flavor% - 2GB RAM, 1 VCPU, 40GB Root disk;
+    * %medium_flavor% - 4GB RAM, 2 VCPUs, 40GB Root disk;
+
+Main URLs
+---------
+
+https://sahara.mirantis.com/jenkins - Sahara CI Jenkins
+https://github.com/stackforge/sahara-ci-config/ - Sahara CI Config repo

etc/scenario/sahara-ci/cdh-5.3.0.yaml

@@ -1,10 +1,10 @@
 clusters:
   - plugin_name: cdh
     plugin_version: 5.3.0
-    image: ubuntu_sahara_cloudera
+    image: %cdh_image%
     node_group_templates:
       - name: worker-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - HDFS_DATANODE
         volumes_per_node: 2
@@ -15,12 +15,12 @@ clusters:
           DATANODE:
             dfs_datanode_du_reserved: 0
       - name: worker-nm
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - YARN_NODEMANAGER
         auto_security_group: true
       - name: worker-nm-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - YARN_NODEMANAGER
           - HDFS_DATANODE
@@ -30,18 +30,18 @@ clusters:
         node_configs:
           *ng_configs
       - name: manager
-        flavor_id: '3'
+        flavor_id: %medium_flavor_id%
         node_processes:
           - CLOUDERA_MANAGER
         auto_security_group: true
       - name: master-core
-        flavor_id: '3'
+        flavor_id: %medium_flavor_id%
         node_processes:
           - HDFS_NAMENODE
           - YARN_RESOURCEMANAGER
         auto_security_group: true
       - name: master-additional
-        flavor_id: '3'
+        flavor_id: %medium_flavor_id%
         node_processes:
           - OOZIE_SERVER
           - YARN_JOBHISTORY
@@ -62,15 +62,7 @@ clusters:
         HDFS:
           dfs_replication: 1
     cluster:
-      name: cdh530
+      name: %cluster_name%
-    scaling:
+    scenario:
-      - operation: add
+      - run_jobs
-        node_group: worker-nm-dn
-        size: 1
-      - operation: resize
-        node_group: worker-nm
-        size: 0
-      - operation: resize
-        node_group: worker-dn
-        size: 0
     edp_jobs_flow: hadoop_2

etc/scenario/sahara-ci/credentials.yaml

@@ -1,9 +1,10 @@
 credentials:
-  os_username: username
+  os_username: %OS_USERNAME%
-  os_password: password
+  os_password: %OS_PASSWORD%
-  os_tenant: tenant
+  os_tenant: %OS_TENANT_NAME%
-  os_auth_url: http://127.0.0.1:5000/v2.0
+  os_auth_url: http://%OPENSTACK_HOST%:5000/v2.0
 
 network:
+  type: %NETWORK%
   private_network: private
   public_network: public

etc/scenario/sahara-ci/edp.yaml

@@ -3,13 +3,16 @@ edp_jobs_flow:
     - type: Pig
       input_datasource:
         type: swift
-        source: etc/edp-examples/edp-pig/top-todoers/data/input
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: hdfs
         destination: /user/hadoop/edp-output
       main_lib:
         type: swift
-        source: etc/edp-examples/edp-pig/top-todoers/example.pig
+        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+      additional_libs:
+        - type: swift
+          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
     - type: MapReduce
       input_datasource:
         type: swift
@@ -46,13 +49,18 @@ edp_jobs_flow:
     - type: Pig
       input_datasource:
         type: swift
-        source: etc/edp-examples/edp-pig/top-todoers/data/input
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: hdfs
         destination: /user/hadoop/edp-output
       main_lib:
         type: swift
-        source: etc/edp-examples/edp-pig/top-todoers/example.pig
+        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+      additional_libs:
+        - type: swift
+          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
+      configs:
+        dfs.replication: 1
     - type: MapReduce
       input_datasource:
         type: swift
@@ -75,14 +83,9 @@ edp_jobs_flow:
         type: hdfs
         destination: /user/hadoop/edp-output
       configs:
+        dfs.replication: 1
         edp.streaming.mapper: /bin/cat
         edp.streaming.reducer: /usr/bin/wc
-    - type: Java
-      additional_libs:
-        - type: database
-          source: etc/edp-examples/hadoop2/edp-java/edp-java.jar
-      configs:
-       edp.java.main_class: org.openstack.sahara.examples.WordCount
   spark_edp:
     - type: Spark
       main_lib:
@@ -92,3 +95,18 @@ edp_jobs_flow:
         edp.java.main_class: org.apache.spark.examples.SparkPi
       args:
         - 4
+  transient:
+    - type: Pig
+      input_datasource:
+        type: swift
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+      output_datasource:
+        type: hdfs
+        destination: /user/hadoop/edp-output
+      main_lib:
+        type: swift
+        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+      additional_libs:
+        - type: swift
+          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
+

etc/scenario/sahara-ci/hdp-1.3.2.yaml

@@ -1,10 +1,10 @@
 clusters:
   - plugin_name: hdp
     plugin_version: 1.3.2
-    image: image_name_for_hdp_1_3_2
+    image: %hdp_image%
     node_group_templates:
       - name: master
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - JOBTRACKER
           - NAMENODE
@@ -13,9 +13,9 @@ clusters:
           - NAGIOS_SERVER
           - AMBARI_SERVER
           - OOZIE_SERVER
-        auto_security_group: true
+        auto_security_group: false
       - name: worker
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - TASKTRACKER
           - DATANODE
@@ -25,20 +25,14 @@ clusters:
           - PIG
         volumes_per_node: 2
         volumes_size: 2
-        auto_security_group: true
+        auto_security_group: false
-        node_configs:
-          &ng_configs
-          DATANODE:
-            dfs_datanode_du_reserved: 0
     cluster_template:
       name: hdp132
       node_group_templates:
         master: 1
         worker: 3
-      HDFS:
-          datanode_du_reserved: 0
     cluster:
-      name: hdp132
+      name: %cluster_name%
     scaling:
       - operation: add
         node_group: worker

etc/scenario/sahara-ci/hdp-2.0.6.yaml

@@ -1,10 +1,10 @@
 clusters:
   - plugin_name: hdp
     plugin_version: 2.0.6
-    image: image_name_for_hdp_2_0_6
+    image: %hdp_two_image%
     node_group_templates:
       - name: master
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - AMBARI_SERVER
           - GANGLIA_SERVER
@@ -17,7 +17,7 @@ clusters:
           - ZOOKEEPER_SERVER
         auto_security_group: true
       - name: worker
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - DATANODE
           - HDFS_CLIENT
@@ -38,7 +38,10 @@ clusters:
       cluster_configs:
         YARN:
           yarn.log-aggregation-enable: false
-        HDFS:
-          datanode_du_reserved: 0
     cluster:
-      name: hdp206
+      name: %cluster_name%
+    scaling:
+      - operation: add
+        node_group: worker
+        size: 1
+    edp_jobs_flow: hadoop_2

etc/scenario/sahara-ci/spark-1.0.0.yaml

@@ -1,16 +1,16 @@
 clusters:
   - plugin_name: spark
     plugin_version: 1.0.0
-    image: image_name_for_spark_1_0_0
+    image: %spark_image%
     node_group_templates:
       - name: master
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - master
           - namenode
         auto_security_group: true
       - name: worker
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - datanode
           - slave
@@ -24,9 +24,7 @@ clusters:
         HDFS:
           dfs.replication: 1
     cluster:
-      name: spark100
+      name: %cluster_name%
-    scaling:
+    scenario:
-      - operation: add
+      - run_jobs
-        node_group: worker
-        size: 1
     edp_jobs_flow: spark_edp

etc/scenario/sahara-ci/transient.yaml

@@ -1,17 +1,13 @@
 clusters:
   - plugin_name: vanilla
     plugin_version: 2.6.0
-    image: image_name_for_vanilla_2_6_0
+    image: %vanilla_two_six_image%
     node_group_templates:
-      - name: transient_ng
+      - name: worker
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - datanode
           - nodemanager
-          - oozie
-          - historyserver
-          - secondarynamenode
-          - resourcemanager
         volumes_per_node: 2
         volumes_size: 2
         auto_security_group: true
@@ -24,19 +20,31 @@ clusters:
             yarn.scheduler.minimum-allocation-mb: 256
             yarn.scheduler.maximum-allocation-mb: 1024
             yarn.nodemanager.vmem-check-enabled: false
+      - name: master
+        flavor_id: %ci_flavor_id%
+        node_processes:
+          - oozie
+          - historyserver
+          - resourcemanager
+          - namenode
+        auto_security_group: true
     cluster_template:
       name: transient
       node_group_templates:
-        transient_ng: 1
+        master: 1
+        worker: 3
       cluster_configs:
         HDFS:
           dfs.replication: 1
         MapReduce:
-          mapreduce.tasktracker.map.tasks.maximum: 16,
+          mapreduce.tasktracker.map.tasks.maximum: 16
           mapreduce.tasktracker.reduce.tasks.maximum: 16
         YARN:
           yarn.resourcemanager.scheduler.class: org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
     cluster:
-      name: transient
+      name: %cluster_name%
       is_transient: true
-    edp_jobs_flow: hadoop_2
+    scenario:
+      - run_jobs
+      - transient
+    edp_jobs_flow: transient

etc/scenario/sahara-ci/vanilla-1.2.1.yaml

@@ -1,10 +1,10 @@
 clusters:
   - plugin_name: vanilla
     plugin_version: 1.2.1
-    image: image_name_for_vanilla_1_2_1
+    image: %vanilla_image%
     node_group_templates:
       - name: worker-tt-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - datanode
           - tasktracker
@@ -12,25 +12,25 @@ clusters:
         volumes_size: 2
         auto_security_group: true
       - name: worker-tt
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - tasktracker
         auto_security_group: true
       - name: worker-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - datanode
         volumes_per_node: 2
         volumes_size: 2
         auto_security_group: true
       - name: master-jt-nn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - namenode
           - jobtracker
         auto_security_group: true
       - name: master-sec-nn-oz
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - oozie
           - secondarynamenode
@@ -49,11 +49,11 @@ clusters:
           dfs.replication: 1
         MapReduce:
           mapred.map.tasks.speculative.execution: False
-          mapred.child.java.opts: -Xmx500m
+          mapred.child.java.opts: -Xmx512m
         general:
           'Enable Swift': True
     cluster:
-      name: vanilla121
+      name: %cluster_name%
     scaling:
       - operation: resize
         node_group: worker-tt-dn

etc/scenario/sahara-ci/vanilla-2.6.0.yaml

@@ -1,10 +1,10 @@
 clusters:
   - plugin_name: vanilla
     plugin_version: 2.6.0
-    image: image_name_for_vanilla_2_6_0
+    image: %vanilla_two_six_image%
     node_group_templates:
       - name: worker-dn-nm
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - datanode
           - nodemanager
@@ -21,14 +21,14 @@ clusters:
             yarn.scheduler.maximum-allocation-mb: 1024
             yarn.nodemanager.vmem-check-enabled: false
       - name: worker-nm
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - nodemanager
         auto_security_group: true
         node_configs:
           *ng_configs
       - name: worker-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - datanode
         volumes_per_node: 2
@@ -37,7 +37,7 @@ clusters:
         node_configs:
           *ng_configs
       - name: master-rm-nn-hvs
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - namenode
           - resourcemanager
@@ -46,7 +46,7 @@ clusters:
         node_configs:
           *ng_configs
       - name: master-oo-hs-sn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
         node_processes:
           - oozie
           - historyserver
@@ -66,7 +66,7 @@ clusters:
         HDFS:
           dfs.replication: 1
     cluster:
-      name: vanilla260
+      name: %cluster_name%
     scaling:
       - operation: resize
         node_group: worker-dn-nm