Merge "Add scenario files for new integration tests"

etc/scenario/sahara-ci/README.rst

@@ -0,0 +1,5 @@
+=========
+Sahara CI
+=========
+
+This files need for sahara-ci. Modifying these files change the behavior tests

etc/scenario/sahara-ci/cdh-5.3.0.yaml

@@ -0,0 +1,76 @@
+clusters:
+  - plugin_name: cdh
+    plugin_version: 5.3.0
+    image: ubuntu_sahara_cloudera
+    node_group_templates:
+      - name: worker-dn
+        flavor_id: '3'
+        node_processes:
+          - HDFS_DATANODE
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+        node_configs:
+          &ng_configs
+          DATANODE:
+            dfs_datanode_du_reserved: 0
+      - name: worker-nm
+        flavor_id: '3'
+        node_processes:
+          - YARN_NODEMANAGER
+        auto_security_group: true
+      - name: worker-nm-dn
+        flavor_id: '3'
+        node_processes:
+          - YARN_NODEMANAGER
+          - HDFS_DATANODE
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+        node_configs:
+          *ng_configs
+      - name: manager
+        flavor_id: '3'
+        node_processes:
+          - CLOUDERA_MANAGER
+        auto_security_group: true
+      - name: master-core
+        flavor_id: '3'
+        node_processes:
+          - HDFS_NAMENODE
+          - YARN_RESOURCEMANAGER
+        auto_security_group: true
+      - name: master-additional
+        flavor_id: '3'
+        node_processes:
+          - OOZIE_SERVER
+          - YARN_JOBHISTORY
+          - HDFS_SECONDARYNAMENODE
+          - HIVE_METASTORE
+          - HIVE_SERVER2
+        auto_security_group: true
+    cluster_template:
+      name: cdh-5-3-0
+      node_group_templates:
+        manager: 1
+        master-core: 1
+        master-additional: 1
+        worker-nm-dn: 1
+        worker-nm: 1
+        worker-dn: 1
+      cluster_configs:
+        HDFS:
+          dfs_replication: 1
+    cluster:
+      name: scenario-cdh
+    scaling:
+      - operation: add
+        node_group: worker-nm-dn
+        size: 1
+      - operation: resize
+        node_group: worker-nm
+        size: 0
+      - operation: resize
+        node_group: worker-dn
+        size: 0
+    edp_jobs_flow: hadoop_2

etc/scenario/sahara-ci/credentials.yaml

@@ -0,0 +1,9 @@
+credentials:
+  os_username: username
+  os_password: password
+  os_tenant: tenant
+  os_auth_url: http://127.0.0.1:5000/v2.0
+
+network:
+  private_network: private
+  public_network: public

etc/scenario/sahara-ci/edp.yaml

@@ -0,0 +1,94 @@
+edp_jobs_flow:
+  hadoop_2:
+    - type: Pig
+      input_datasource:
+        type: swift
+        source: etc/edp-examples/edp-pig/top-todoers/data/input
+      output_datasource:
+        type: hdfs
+        destination: /user/hadoop/edp-output
+      main_lib:
+        type: swift
+        source: etc/edp-examples/edp-pig/top-todoers/example.pig
+    - type: MapReduce
+      input_datasource:
+        type: swift
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+      output_datasource:
+        type: hdfs
+        destination: /user/hadoop/edp-output
+      additional_libs:
+        - type: database
+          source:  etc/edp-examples/edp-mapreduce/edp-mapreduce.jar
+      configs:
+        mapred.mapper.class: org.apache.oozie.example.SampleMapper
+        mapred.reducer.class: org.apache.oozie.example.SampleReducer
+    - type: MapReduce.Streaming
+      input_datasource:
+        type: swift
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+      output_datasource:
+        type: hdfs
+        destination: /user/hadoop/edp-output
+      configs:
+        edp.streaming.mapper: /bin/cat
+        edp.streaming.reducer: /usr/bin/wc
+    - type: Java
+      additional_libs:
+        - type: database
+          source: etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.6.0.jar
+      configs:
+       edp.java.main_class: org.apache.hadoop.examples.QuasiMonteCarlo
+      args:
+        - 10
+        - 10
+  hadoop_1:
+    - type: Pig
+      input_datasource:
+        type: swift
+        source: etc/edp-examples/edp-pig/top-todoers/data/input
+      output_datasource:
+        type: hdfs
+        destination: /user/hadoop/edp-output
+      main_lib:
+        type: swift
+        source: etc/edp-examples/edp-pig/top-todoers/example.pig
+    - type: MapReduce
+      input_datasource:
+        type: swift
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+      output_datasource:
+        type: hdfs
+        destination: /user/hadoop/edp-output
+      additional_libs:
+        - type: database
+          source:  etc/edp-examples/edp-mapreduce/edp-mapreduce.jar
+      configs:
+        dfs.replication: 1
+        mapred.mapper.class: org.apache.oozie.example.SampleMapper
+        mapred.reducer.class: org.apache.oozie.example.SampleReducer
+    - type: MapReduce.Streaming
+      input_datasource:
+        type: swift
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+      output_datasource:
+        type: hdfs
+        destination: /user/hadoop/edp-output
+      configs:
+        edp.streaming.mapper: /bin/cat
+        edp.streaming.reducer: /usr/bin/wc
+    - type: Java
+      additional_libs:
+        - type: database
+          source: etc/edp-examples/hadoop2/edp-java/edp-java.jar
+      configs:
+       edp.java.main_class: org.openstack.sahara.examples.WordCount
+  spark_edp:
+    - type: Spark
+      main_lib:
+        type: database
+        source: etc/edp-examples/edp-spark/spark-example.jar
+      configs:
+        edp.java.main_class: org.apache.spark.examples.SparkPi
+      args:
+        - 4

etc/scenario/sahara-ci/hdp-1.3.2.yaml

@@ -0,0 +1,46 @@
+clusters:
+  - plugin_name: hdp
+    plugin_version: 1.3.2
+    image: image_name_for_hdp_1_3_2
+    node_group_templates:
+      - name: master
+        flavor_id: '3'
+        node_processes:
+          - JOBTRACKER
+          - NAMENODE
+          - SECONDARY_NAMENODE
+          - GANGLIA_SERVER
+          - NAGIOS_SERVER
+          - AMBARI_SERVER
+          - OOZIE_SERVER
+        auto_security_group: true
+      - name: worker
+        flavor_id: '3'
+        node_processes:
+          - TASKTRACKER
+          - DATANODE
+          - HDFS_CLIENT
+          - MAPREDUCE_CLIENT
+          - OOZIE_CLIENT
+          - PIG
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+        node_configs:
+          &ng_configs
+          DATANODE:
+            dfs_datanode_du_reserved: 0
+    cluster_template:
+      name: hdp-1-3-2
+      node_group_templates:
+        master: 1
+        worker: 3
+      HDFS:
+          datanode_du_reserved: 0
+    cluster:
+      name: scenario-hdp1
+    scaling:
+      - operation: add
+        node_group: worker
+        size: 1
+    edp_jobs_flow: hadoop_1

etc/scenario/sahara-ci/hdp-2.0.6.yaml

@@ -0,0 +1,44 @@
+clusters:
+  - plugin_name: hdp
+    plugin_version: 2.0.6
+    image: image_name_for_hdp_2_0_6
+    node_group_templates:
+      - name: master
+        flavor_id: '3'
+        node_processes:
+          - AMBARI_SERVER
+          - GANGLIA_SERVER
+          - HISTORYSERVER
+          - NAGIOS_SERVER
+          - NAMENODE
+          - OOZIE_SERVER
+          - RESOURCEMANAGER
+          - SECONDARY_NAMENODE
+          - ZOOKEEPER_SERVER
+        auto_security_group: true
+      - name: worker
+        flavor_id: '3'
+        node_processes:
+          - DATANODE
+          - HDFS_CLIENT
+          - MAPREDUCE2_CLIENT
+          - NODEMANAGER
+          - OOZIE_CLIENT
+          - PIG
+          - YARN_CLIENT
+          - ZOOKEEPER_CLIENT
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+    cluster_template:
+      name: hdp-2-0-6
+      node_group_templates:
+        master: 1
+        worker: 3
+      cluster_configs:
+        YARN:
+          yarn.log-aggregation-enable: false
+        HDFS:
+          datanode_du_reserved: 0
+    cluster:
+      name: scenario-hdp2

etc/scenario/sahara-ci/spark-1.0.0.yaml

@@ -0,0 +1,32 @@
+clusters:
+  - plugin_name: spark
+    plugin_version: 1.0.0
+    image: image_name_for_spark_1_0_0
+    node_group_templates:
+      - name: master
+        flavor_id: '3'
+        node_processes:
+          - master
+          - namenode
+        auto_security_group: true
+      - name: worker
+        flavor_id: '3'
+        node_processes:
+          - datanode
+          - slave
+        auto_security_group: true
+    cluster_template:
+      name: spark-1-0-0
+      node_group_templates:
+        master: 1
+        worker: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+    cluster:
+      name: scenario-spark
+    scaling:
+      - operation: add
+        node_group: worker
+        size: 1
+    edp_jobs_flow: spark_edp

etc/scenario/sahara-ci/transient.yaml

@@ -0,0 +1,42 @@
+clusters:
+  - plugin_name: vanilla
+    plugin_version: 2.6.0
+    image: image_name_for_vanilla_2_6_0
+    node_group_templates:
+      - name: transient_ng
+        flavor_id: '3'
+        node_processes:
+          - datanode
+          - nodemanager
+          - oozie
+          - historyserver
+          - secondarynamenode
+          - resourcemanager
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+        node_configs:
+          &ng_configs
+          MapReduce:
+            yarn.app.mapreduce.am.resource.mb: 256
+            yarn.app.mapreduce.am.command-opts: -Xmx256m
+          YARN:
+            yarn.scheduler.minimum-allocation-mb: 256
+            yarn.scheduler.maximum-allocation-mb: 1024
+            yarn.nodemanager.vmem-check-enabled: false
+    cluster_template:
+      name: vanilla-2-6
+      node_group_templates:
+        transient_ng: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+        MapReduce:
+          mapreduce.tasktracker.map.tasks.maximum: 16,
+          mapreduce.tasktracker.reduce.tasks.maximum: 16
+        YARN:
+          yarn.resourcemanager.scheduler.class: org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
+    cluster:
+      name: scenario-transient
+      is_transient: true
+    edp_jobs_flow: hadoop_2

etc/scenario/sahara-ci/vanilla-1.2.1.yaml

@@ -0,0 +1,73 @@
+clusters:
+  - plugin_name: vanilla
+    plugin_version: 1.2.1
+    image: image_name_for_vanilla_1_2_1
+    node_group_templates:
+      - name: worker-tt-dn
+        flavor_id: '3'
+        node_processes:
+          - datanode
+          - tasktracker
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+      - name: worker-tt
+        flavor_id: '3'
+        node_processes:
+          - tasktracker
+        auto_security_group: true
+      - name: worker-dn
+        flavor_id: '3'
+        node_processes:
+          - datanode
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+      - name: master-jt-nn
+        flavor_id: '3'
+        node_processes:
+          - namenode
+          - jobtracker
+        auto_security_group: true
+      - name: master-sec-nn-oz
+        flavor_id: '3'
+        node_processes:
+          - oozie
+          - secondarynamenode
+        auto_security_group: true
+
+    cluster_template:
+      name: vanilla-1-2-1
+      node_group_templates:
+        master-sec-nn-oz: 1
+        master-jt-nn: 1
+        worker-tt: 1
+        worker-tt-dn: 2
+        worker-dn: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+        MapReduce:
+          mapred.map.tasks.speculative.execution: False
+          mapred.child.java.opts: -Xmx500m
+        general:
+          'Enable Swift': True
+    cluster:
+      name: scenario-vanilla1
+    scaling:
+      - operation: resize
+        node_group: worker-tt-dn
+        size: 1
+      - operation: resize
+        node_group: worker-dn
+        size: 0
+      - operation: resize
+        node_group: worker-tt
+        size: 0
+      - operation: add
+        node_group: worker-tt
+        size: 1
+      - operation: add
+        node_group: worker-dn
+        size: 1
+    edp_jobs_flow: hadoop_1

etc/scenario/sahara-ci/vanilla-2.6.0.yaml

@@ -0,0 +1,86 @@
+clusters:
+  - plugin_name: vanilla
+    plugin_version: 2.6.0
+    image: image_name_for_vanilla_2_6_0
+    node_group_templates:
+      - name: worker-dn-nm
+        flavor_id: '3'
+        node_processes:
+          - datanode
+          - nodemanager
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+        node_configs:
+          &ng_configs
+          MapReduce:
+            yarn.app.mapreduce.am.resource.mb: 256
+            yarn.app.mapreduce.am.command-opts: -Xmx256m
+          YARN:
+            yarn.scheduler.minimum-allocation-mb: 256
+            yarn.scheduler.maximum-allocation-mb: 1024
+            yarn.nodemanager.vmem-check-enabled: false
+      - name: worker-nm
+        flavor_id: '3'
+        node_processes:
+          - nodemanager
+        auto_security_group: true
+        node_configs:
+          *ng_configs
+      - name: worker-dn
+        flavor_id: '3'
+        node_processes:
+          - datanode
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+        node_configs:
+          *ng_configs
+      - name: master-rm-nn-hvs
+        flavor_id: '3'
+        node_processes:
+          - namenode
+          - resourcemanager
+          - hiveserver
+        auto_security_group: true
+        node_configs:
+          *ng_configs
+      - name: master-oo-hs-sn
+        flavor_id: '3'
+        node_processes:
+          - oozie
+          - historyserver
+          - secondarynamenode
+        auto_security_group: true
+        node_configs:
+          *ng_configs
+    cluster_template:
+      name: vanilla-2-6
+      node_group_templates:
+        master-rm-nn-hvs: 1
+        master-oo-hs-sn: 1
+        worker-dn-nm: 2
+        worker-dn: 1
+        worker-nm: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+    cluster:
+      name: scenario-vanilla2
+    scaling:
+      - operation: resize
+        node_group: worker-dn-nm
+        size: 1
+      - operation: resize
+        node_group: worker-dn
+        size: 0
+      - operation: resize
+        node_group: worker-nm
+        size: 0
+      - operation: add
+        node_group: worker-dn
+        size: 1
+      - operation: add
+        node_group: worker-nm
+        size: 1
+    edp_jobs_flow: hadoop_2