Put in Sahara repo actual scenario files

Now scenario files placed in Sahara repo aren't equal to scenario files which actually is using in Sahara CI. Also we want to use scenario files from Sahara repo, instead Sahara CI repo, so we need to update this files. Change-Id: Id6adb8f89934ffe4ceb85cf7c12359dca8eff09e
2015-04-10 19:51:54 +03:00 · 2015-04-10 19:51:54 +03:00 · 5e2980ec05
commit 5e2980ec05
parent 5b79d2ebfe
10 changed files with 120 additions and 84 deletions
--- a/etc/scenario/sahara-ci/README.rst
+++ b/etc/scenario/sahara-ci/README.rst
@ -5,3 +5,25 @@ Sahara CI
 The files in this directory are needed for the sahara continuous
 integration tests. Modifying these files will change the behavior of the
 tests.
+
+Details
+-------
+
+Key values:
+
+* %OS_USERNAME%, %OS_PASSWORD%, %OS_TENANT_NAME%, %OPENSTACK_HOST% - OpenStack credentials;
+* %NETWORK% - network type (neutron or nova-network);
+* %cluster_name% - name of cluster, which generating from $HOST-$ZUUL_CHANGE-$CLUSTER_HASH. Where:
+    * $HOST - host id (c1 - with neutron, c2 - with nova-network);
+    * $ZUUL_CHANGE - change number;
+    * $CLUSTER_HASH - hash, which generating for each cluster by using "uuid" python module;
+* %{plugin}_image% - name of image for each plugin;
+* flavor ids:
+    * %ci_flavor% - 2GB RAM, 1 VCPU, 40GB Root disk;
+    * %medium_flavor% - 4GB RAM, 2 VCPUs, 40GB Root disk;
+
+Main URLs
+---------
+
+https://sahara.mirantis.com/jenkins - Sahara CI Jenkins
+https://github.com/stackforge/sahara-ci-config/ - Sahara CI Config repo
--- a/etc/scenario/sahara-ci/cdh-5.3.0.yaml
+++ b/etc/scenario/sahara-ci/cdh-5.3.0.yaml
@ -1,10 +1,10 @@
 clusters:
  - plugin_name: cdh
    plugin_version: 5.3.0
-    image: ubuntu_sahara_cloudera
+    image: %cdh_image%
    node_group_templates:
      - name: worker-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - HDFS_DATANODE
        volumes_per_node: 2
@ -15,12 +15,12 @@ clusters:
          DATANODE:
            dfs_datanode_du_reserved: 0
      - name: worker-nm
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - YARN_NODEMANAGER
        auto_security_group: true
      - name: worker-nm-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - YARN_NODEMANAGER
          - HDFS_DATANODE
@ -30,18 +30,18 @@ clusters:
        node_configs:
          *ng_configs
      - name: manager
-        flavor_id: '3'
+        flavor_id: %medium_flavor_id%
        node_processes:
          - CLOUDERA_MANAGER
        auto_security_group: true
      - name: master-core
-        flavor_id: '3'
+        flavor_id: %medium_flavor_id%
        node_processes:
          - HDFS_NAMENODE
          - YARN_RESOURCEMANAGER
        auto_security_group: true
      - name: master-additional
-        flavor_id: '3'
+        flavor_id: %medium_flavor_id%
        node_processes:
          - OOZIE_SERVER
          - YARN_JOBHISTORY
@ -62,15 +62,7 @@ clusters:
        HDFS:
          dfs_replication: 1
    cluster:
-      name: cdh530
-    scaling:
-      - operation: add
-        node_group: worker-nm-dn
-        size: 1
-      - operation: resize
-        node_group: worker-nm
-        size: 0
-      - operation: resize
-        node_group: worker-dn
-        size: 0
+      name: %cluster_name%
+    scenario:
+      - run_jobs
    edp_jobs_flow: hadoop_2
--- a/etc/scenario/sahara-ci/credentials.yaml
+++ b/etc/scenario/sahara-ci/credentials.yaml
@ -1,9 +1,10 @@
 credentials:
-  os_username: username
-  os_password: password
-  os_tenant: tenant
-  os_auth_url: http://127.0.0.1:5000/v2.0
+  os_username: %OS_USERNAME%
+  os_password: %OS_PASSWORD%
+  os_tenant: %OS_TENANT_NAME%
+  os_auth_url: http://%OPENSTACK_HOST%:5000/v2.0

 network:
+  type: %NETWORK%
  private_network: private
  public_network: public
--- a/etc/scenario/sahara-ci/edp.yaml
+++ b/etc/scenario/sahara-ci/edp.yaml
@ -3,13 +3,16 @@ edp_jobs_flow:
    - type: Pig
      input_datasource:
        type: swift
-        source: etc/edp-examples/edp-pig/top-todoers/data/input
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
      output_datasource:
        type: hdfs
        destination: /user/hadoop/edp-output
      main_lib:
        type: swift
-        source: etc/edp-examples/edp-pig/top-todoers/example.pig
+        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+      additional_libs:
+        - type: swift
+          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
    - type: MapReduce
      input_datasource:
        type: swift
@ -46,13 +49,18 @@ edp_jobs_flow:
    - type: Pig
      input_datasource:
        type: swift
-        source: etc/edp-examples/edp-pig/top-todoers/data/input
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
      output_datasource:
        type: hdfs
        destination: /user/hadoop/edp-output
      main_lib:
        type: swift
-        source: etc/edp-examples/edp-pig/top-todoers/example.pig
+        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+      additional_libs:
+        - type: swift
+          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
+      configs:
+        dfs.replication: 1
    - type: MapReduce
      input_datasource:
        type: swift
@ -75,14 +83,9 @@ edp_jobs_flow:
        type: hdfs
        destination: /user/hadoop/edp-output
      configs:
+        dfs.replication: 1
        edp.streaming.mapper: /bin/cat
        edp.streaming.reducer: /usr/bin/wc
-    - type: Java
-      additional_libs:
-        - type: database
-          source: etc/edp-examples/hadoop2/edp-java/edp-java.jar
-      configs:
-       edp.java.main_class: org.openstack.sahara.examples.WordCount
  spark_edp:
    - type: Spark
      main_lib:
@ -92,3 +95,18 @@ edp_jobs_flow:
        edp.java.main_class: org.apache.spark.examples.SparkPi
      args:
        - 4
+  transient:
+    - type: Pig
+      input_datasource:
+        type: swift
+        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+      output_datasource:
+        type: hdfs
+        destination: /user/hadoop/edp-output
+      main_lib:
+        type: swift
+        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+      additional_libs:
+        - type: swift
+          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
+
--- a/etc/scenario/sahara-ci/hdp-1.3.2.yaml
+++ b/etc/scenario/sahara-ci/hdp-1.3.2.yaml
@ -1,10 +1,10 @@
 clusters:
  - plugin_name: hdp
    plugin_version: 1.3.2
-    image: image_name_for_hdp_1_3_2
+    image: %hdp_image%
    node_group_templates:
      - name: master
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - JOBTRACKER
          - NAMENODE
@ -13,9 +13,9 @@ clusters:
          - NAGIOS_SERVER
          - AMBARI_SERVER
          - OOZIE_SERVER
-        auto_security_group: true
+        auto_security_group: false
      - name: worker
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - TASKTRACKER
          - DATANODE
@ -25,20 +25,14 @@ clusters:
          - PIG
        volumes_per_node: 2
        volumes_size: 2
-        auto_security_group: true
-        node_configs:
-          &ng_configs
-          DATANODE:
-            dfs_datanode_du_reserved: 0
+        auto_security_group: false
    cluster_template:
      name: hdp132
      node_group_templates:
        master: 1
        worker: 3
-      HDFS:
-          datanode_du_reserved: 0
    cluster:
-      name: hdp132
+      name: %cluster_name%
    scaling:
      - operation: add
        node_group: worker
--- a/etc/scenario/sahara-ci/hdp-2.0.6.yaml
+++ b/etc/scenario/sahara-ci/hdp-2.0.6.yaml
@ -1,10 +1,10 @@
 clusters:
  - plugin_name: hdp
    plugin_version: 2.0.6
-    image: image_name_for_hdp_2_0_6
+    image: %hdp_two_image%
    node_group_templates:
      - name: master
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - AMBARI_SERVER
          - GANGLIA_SERVER
@ -17,7 +17,7 @@ clusters:
          - ZOOKEEPER_SERVER
        auto_security_group: true
      - name: worker
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - DATANODE
          - HDFS_CLIENT
@ -38,7 +38,10 @@ clusters:
      cluster_configs:
        YARN:
          yarn.log-aggregation-enable: false
-        HDFS:
-          datanode_du_reserved: 0
    cluster:
-      name: hdp206
+      name: %cluster_name%
+    scaling:
+      - operation: add
+        node_group: worker
+        size: 1
+    edp_jobs_flow: hadoop_2
--- a/etc/scenario/sahara-ci/spark-1.0.0.yaml
+++ b/etc/scenario/sahara-ci/spark-1.0.0.yaml
@ -1,16 +1,16 @@
 clusters:
  - plugin_name: spark
    plugin_version: 1.0.0
-    image: image_name_for_spark_1_0_0
+    image: %spark_image%
    node_group_templates:
      - name: master
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - master
          - namenode
        auto_security_group: true
      - name: worker
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - datanode
          - slave
@ -24,9 +24,7 @@ clusters:
        HDFS:
          dfs.replication: 1
    cluster:
-      name: spark100
-    scaling:
-      - operation: add
-        node_group: worker
-        size: 1
+      name: %cluster_name%
+    scenario:
+      - run_jobs
    edp_jobs_flow: spark_edp
--- a/etc/scenario/sahara-ci/transient.yaml
+++ b/etc/scenario/sahara-ci/transient.yaml
@ -1,17 +1,13 @@
 clusters:
  - plugin_name: vanilla
    plugin_version: 2.6.0
-    image: image_name_for_vanilla_2_6_0
+    image: %vanilla_two_six_image%
    node_group_templates:
-      - name: transient_ng
-        flavor_id: '3'
+      - name: worker
+        flavor_id: %ci_flavor_id%
        node_processes:
          - datanode
          - nodemanager
-          - oozie
-          - historyserver
-          - secondarynamenode
-          - resourcemanager
        volumes_per_node: 2
        volumes_size: 2
        auto_security_group: true
@ -24,19 +20,31 @@ clusters:
            yarn.scheduler.minimum-allocation-mb: 256
            yarn.scheduler.maximum-allocation-mb: 1024
            yarn.nodemanager.vmem-check-enabled: false
+      - name: master
+        flavor_id: %ci_flavor_id%
+        node_processes:
+          - oozie
+          - historyserver
+          - resourcemanager
+          - namenode
+        auto_security_group: true
    cluster_template:
      name: transient
      node_group_templates:
-        transient_ng: 1
+        master: 1
+        worker: 3
      cluster_configs:
        HDFS:
          dfs.replication: 1
        MapReduce:
-          mapreduce.tasktracker.map.tasks.maximum: 16,
+          mapreduce.tasktracker.map.tasks.maximum: 16
          mapreduce.tasktracker.reduce.tasks.maximum: 16
        YARN:
          yarn.resourcemanager.scheduler.class: org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
    cluster:
-      name: transient
+      name: %cluster_name%
      is_transient: true
-    edp_jobs_flow: hadoop_2
+    scenario:
+      - run_jobs
+      - transient
+    edp_jobs_flow: transient
--- a/etc/scenario/sahara-ci/vanilla-1.2.1.yaml
+++ b/etc/scenario/sahara-ci/vanilla-1.2.1.yaml
@ -1,10 +1,10 @@
 clusters:
  - plugin_name: vanilla
    plugin_version: 1.2.1
-    image: image_name_for_vanilla_1_2_1
+    image: %vanilla_image%
    node_group_templates:
      - name: worker-tt-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - datanode
          - tasktracker
@ -12,25 +12,25 @@ clusters:
        volumes_size: 2
        auto_security_group: true
      - name: worker-tt
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - tasktracker
        auto_security_group: true
      - name: worker-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - datanode
        volumes_per_node: 2
        volumes_size: 2
        auto_security_group: true
      - name: master-jt-nn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - namenode
          - jobtracker
        auto_security_group: true
      - name: master-sec-nn-oz
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - oozie
          - secondarynamenode
@ -49,11 +49,11 @@ clusters:
          dfs.replication: 1
        MapReduce:
          mapred.map.tasks.speculative.execution: False
-          mapred.child.java.opts: -Xmx500m
+          mapred.child.java.opts: -Xmx512m
        general:
          'Enable Swift': True
    cluster:
-      name: vanilla121
+      name: %cluster_name%
    scaling:
      - operation: resize
        node_group: worker-tt-dn
--- a/etc/scenario/sahara-ci/vanilla-2.6.0.yaml
+++ b/etc/scenario/sahara-ci/vanilla-2.6.0.yaml
@ -1,10 +1,10 @@
 clusters:
  - plugin_name: vanilla
    plugin_version: 2.6.0
-    image: image_name_for_vanilla_2_6_0
+    image: %vanilla_two_six_image%
    node_group_templates:
      - name: worker-dn-nm
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - datanode
          - nodemanager
@ -21,14 +21,14 @@ clusters:
            yarn.scheduler.maximum-allocation-mb: 1024
            yarn.nodemanager.vmem-check-enabled: false
      - name: worker-nm
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - nodemanager
        auto_security_group: true
        node_configs:
          *ng_configs
      - name: worker-dn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - datanode
        volumes_per_node: 2
@ -37,7 +37,7 @@ clusters:
        node_configs:
          *ng_configs
      - name: master-rm-nn-hvs
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - namenode
          - resourcemanager
@ -46,7 +46,7 @@ clusters:
        node_configs:
          *ng_configs
      - name: master-oo-hs-sn
-        flavor_id: '3'
+        flavor_id: %ci_flavor_id%
        node_processes:
          - oozie
          - historyserver
@ -66,7 +66,7 @@ clusters:
        HDFS:
          dfs.replication: 1
    cluster:
-      name: vanilla260
+      name: %cluster_name%
    scaling:
      - operation: resize
        node_group: worker-dn-nm