diff --git a/etc/scenario/sahara-ci/README.rst b/etc/scenario/sahara-ci/README.rst new file mode 100644 index 0000000000..a3c5687f6b --- /dev/null +++ b/etc/scenario/sahara-ci/README.rst @@ -0,0 +1,5 @@ +========= +Sahara CI +========= + +This files need for sahara-ci. Modifying these files change the behavior tests \ No newline at end of file diff --git a/etc/scenario/sahara-ci/cdh-5.3.0.yaml b/etc/scenario/sahara-ci/cdh-5.3.0.yaml new file mode 100644 index 0000000000..d0b2d945ab --- /dev/null +++ b/etc/scenario/sahara-ci/cdh-5.3.0.yaml @@ -0,0 +1,76 @@ +clusters: + - plugin_name: cdh + plugin_version: 5.3.0 + image: ubuntu_sahara_cloudera + node_group_templates: + - name: worker-dn + flavor_id: '3' + node_processes: + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + DATANODE: + dfs_datanode_du_reserved: 0 + - name: worker-nm + flavor_id: '3' + node_processes: + - YARN_NODEMANAGER + auto_security_group: true + - name: worker-nm-dn + flavor_id: '3' + node_processes: + - YARN_NODEMANAGER + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + *ng_configs + - name: manager + flavor_id: '3' + node_processes: + - CLOUDERA_MANAGER + auto_security_group: true + - name: master-core + flavor_id: '3' + node_processes: + - HDFS_NAMENODE + - YARN_RESOURCEMANAGER + auto_security_group: true + - name: master-additional + flavor_id: '3' + node_processes: + - OOZIE_SERVER + - YARN_JOBHISTORY + - HDFS_SECONDARYNAMENODE + - HIVE_METASTORE + - HIVE_SERVER2 + auto_security_group: true + cluster_template: + name: cdh-5-3-0 + node_group_templates: + manager: 1 + master-core: 1 + master-additional: 1 + worker-nm-dn: 1 + worker-nm: 1 + worker-dn: 1 + cluster_configs: + HDFS: + dfs_replication: 1 + cluster: + name: scenario-cdh + scaling: + - operation: add + node_group: worker-nm-dn + size: 1 + - operation: resize + node_group: worker-nm + size: 0 + - operation: resize + node_group: worker-dn + size: 0 + edp_jobs_flow: hadoop_2 diff --git a/etc/scenario/sahara-ci/credentials.yaml b/etc/scenario/sahara-ci/credentials.yaml new file mode 100644 index 0000000000..286c43b5a0 --- /dev/null +++ b/etc/scenario/sahara-ci/credentials.yaml @@ -0,0 +1,9 @@ +credentials: + os_username: username + os_password: password + os_tenant: tenant + os_auth_url: http://127.0.0.1:5000/v2.0 + +network: + private_network: private + public_network: public diff --git a/etc/scenario/sahara-ci/edp.yaml b/etc/scenario/sahara-ci/edp.yaml new file mode 100644 index 0000000000..06992b9cfe --- /dev/null +++ b/etc/scenario/sahara-ci/edp.yaml @@ -0,0 +1,94 @@ +edp_jobs_flow: + hadoop_2: + - type: Pig + input_datasource: + type: swift + source: etc/edp-examples/edp-pig/top-todoers/data/input + output_datasource: + type: hdfs + destination: /user/hadoop/edp-output + main_lib: + type: swift + source: etc/edp-examples/edp-pig/top-todoers/example.pig + - type: MapReduce + input_datasource: + type: swift + source: etc/edp-examples/edp-pig/trim-spaces/data/input + output_datasource: + type: hdfs + destination: /user/hadoop/edp-output + additional_libs: + - type: database + source: etc/edp-examples/edp-mapreduce/edp-mapreduce.jar + configs: + mapred.mapper.class: org.apache.oozie.example.SampleMapper + mapred.reducer.class: org.apache.oozie.example.SampleReducer + - type: MapReduce.Streaming + input_datasource: + type: swift + source: etc/edp-examples/edp-pig/trim-spaces/data/input + output_datasource: + type: hdfs + destination: /user/hadoop/edp-output + configs: + edp.streaming.mapper: /bin/cat + edp.streaming.reducer: /usr/bin/wc + - type: Java + additional_libs: + - type: database + source: etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.6.0.jar + configs: + edp.java.main_class: org.apache.hadoop.examples.QuasiMonteCarlo + args: + - 10 + - 10 + hadoop_1: + - type: Pig + input_datasource: + type: swift + source: etc/edp-examples/edp-pig/top-todoers/data/input + output_datasource: + type: hdfs + destination: /user/hadoop/edp-output + main_lib: + type: swift + source: etc/edp-examples/edp-pig/top-todoers/example.pig + - type: MapReduce + input_datasource: + type: swift + source: etc/edp-examples/edp-pig/trim-spaces/data/input + output_datasource: + type: hdfs + destination: /user/hadoop/edp-output + additional_libs: + - type: database + source: etc/edp-examples/edp-mapreduce/edp-mapreduce.jar + configs: + dfs.replication: 1 + mapred.mapper.class: org.apache.oozie.example.SampleMapper + mapred.reducer.class: org.apache.oozie.example.SampleReducer + - type: MapReduce.Streaming + input_datasource: + type: swift + source: etc/edp-examples/edp-pig/trim-spaces/data/input + output_datasource: + type: hdfs + destination: /user/hadoop/edp-output + configs: + edp.streaming.mapper: /bin/cat + edp.streaming.reducer: /usr/bin/wc + - type: Java + additional_libs: + - type: database + source: etc/edp-examples/hadoop2/edp-java/edp-java.jar + configs: + edp.java.main_class: org.openstack.sahara.examples.WordCount + spark_edp: + - type: Spark + main_lib: + type: database + source: etc/edp-examples/edp-spark/spark-example.jar + configs: + edp.java.main_class: org.apache.spark.examples.SparkPi + args: + - 4 diff --git a/etc/scenario/sahara-ci/hdp-1.3.2.yaml b/etc/scenario/sahara-ci/hdp-1.3.2.yaml new file mode 100644 index 0000000000..0e4c85514a --- /dev/null +++ b/etc/scenario/sahara-ci/hdp-1.3.2.yaml @@ -0,0 +1,46 @@ +clusters: + - plugin_name: hdp + plugin_version: 1.3.2 + image: image_name_for_hdp_1_3_2 + node_group_templates: + - name: master + flavor_id: '3' + node_processes: + - JOBTRACKER + - NAMENODE + - SECONDARY_NAMENODE + - GANGLIA_SERVER + - NAGIOS_SERVER + - AMBARI_SERVER + - OOZIE_SERVER + auto_security_group: true + - name: worker + flavor_id: '3' + node_processes: + - TASKTRACKER + - DATANODE + - HDFS_CLIENT + - MAPREDUCE_CLIENT + - OOZIE_CLIENT + - PIG + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + DATANODE: + dfs_datanode_du_reserved: 0 + cluster_template: + name: hdp-1-3-2 + node_group_templates: + master: 1 + worker: 3 + HDFS: + datanode_du_reserved: 0 + cluster: + name: scenario-hdp1 + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: hadoop_1 diff --git a/etc/scenario/sahara-ci/hdp-2.0.6.yaml b/etc/scenario/sahara-ci/hdp-2.0.6.yaml new file mode 100644 index 0000000000..d34ec1904e --- /dev/null +++ b/etc/scenario/sahara-ci/hdp-2.0.6.yaml @@ -0,0 +1,44 @@ +clusters: + - plugin_name: hdp + plugin_version: 2.0.6 + image: image_name_for_hdp_2_0_6 + node_group_templates: + - name: master + flavor_id: '3' + node_processes: + - AMBARI_SERVER + - GANGLIA_SERVER + - HISTORYSERVER + - NAGIOS_SERVER + - NAMENODE + - OOZIE_SERVER + - RESOURCEMANAGER + - SECONDARY_NAMENODE + - ZOOKEEPER_SERVER + auto_security_group: true + - name: worker + flavor_id: '3' + node_processes: + - DATANODE + - HDFS_CLIENT + - MAPREDUCE2_CLIENT + - NODEMANAGER + - OOZIE_CLIENT + - PIG + - YARN_CLIENT + - ZOOKEEPER_CLIENT + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + cluster_template: + name: hdp-2-0-6 + node_group_templates: + master: 1 + worker: 3 + cluster_configs: + YARN: + yarn.log-aggregation-enable: false + HDFS: + datanode_du_reserved: 0 + cluster: + name: scenario-hdp2 diff --git a/etc/scenario/sahara-ci/spark-1.0.0.yaml b/etc/scenario/sahara-ci/spark-1.0.0.yaml new file mode 100644 index 0000000000..efc8639afa --- /dev/null +++ b/etc/scenario/sahara-ci/spark-1.0.0.yaml @@ -0,0 +1,32 @@ +clusters: + - plugin_name: spark + plugin_version: 1.0.0 + image: image_name_for_spark_1_0_0 + node_group_templates: + - name: master + flavor_id: '3' + node_processes: + - master + - namenode + auto_security_group: true + - name: worker + flavor_id: '3' + node_processes: + - datanode + - slave + auto_security_group: true + cluster_template: + name: spark-1-0-0 + node_group_templates: + master: 1 + worker: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: scenario-spark + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: spark_edp diff --git a/etc/scenario/sahara-ci/transient.yaml b/etc/scenario/sahara-ci/transient.yaml new file mode 100644 index 0000000000..13f2363da7 --- /dev/null +++ b/etc/scenario/sahara-ci/transient.yaml @@ -0,0 +1,42 @@ +clusters: + - plugin_name: vanilla + plugin_version: 2.6.0 + image: image_name_for_vanilla_2_6_0 + node_group_templates: + - name: transient_ng + flavor_id: '3' + node_processes: + - datanode + - nodemanager + - oozie + - historyserver + - secondarynamenode + - resourcemanager + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + MapReduce: + yarn.app.mapreduce.am.resource.mb: 256 + yarn.app.mapreduce.am.command-opts: -Xmx256m + YARN: + yarn.scheduler.minimum-allocation-mb: 256 + yarn.scheduler.maximum-allocation-mb: 1024 + yarn.nodemanager.vmem-check-enabled: false + cluster_template: + name: vanilla-2-6 + node_group_templates: + transient_ng: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + MapReduce: + mapreduce.tasktracker.map.tasks.maximum: 16, + mapreduce.tasktracker.reduce.tasks.maximum: 16 + YARN: + yarn.resourcemanager.scheduler.class: org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler + cluster: + name: scenario-transient + is_transient: true + edp_jobs_flow: hadoop_2 \ No newline at end of file diff --git a/etc/scenario/sahara-ci/vanilla-1.2.1.yaml b/etc/scenario/sahara-ci/vanilla-1.2.1.yaml new file mode 100644 index 0000000000..4ec12070b7 --- /dev/null +++ b/etc/scenario/sahara-ci/vanilla-1.2.1.yaml @@ -0,0 +1,73 @@ +clusters: + - plugin_name: vanilla + plugin_version: 1.2.1 + image: image_name_for_vanilla_1_2_1 + node_group_templates: + - name: worker-tt-dn + flavor_id: '3' + node_processes: + - datanode + - tasktracker + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + - name: worker-tt + flavor_id: '3' + node_processes: + - tasktracker + auto_security_group: true + - name: worker-dn + flavor_id: '3' + node_processes: + - datanode + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + - name: master-jt-nn + flavor_id: '3' + node_processes: + - namenode + - jobtracker + auto_security_group: true + - name: master-sec-nn-oz + flavor_id: '3' + node_processes: + - oozie + - secondarynamenode + auto_security_group: true + + cluster_template: + name: vanilla-1-2-1 + node_group_templates: + master-sec-nn-oz: 1 + master-jt-nn: 1 + worker-tt: 1 + worker-tt-dn: 2 + worker-dn: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + MapReduce: + mapred.map.tasks.speculative.execution: False + mapred.child.java.opts: -Xmx500m + general: + 'Enable Swift': True + cluster: + name: scenario-vanilla1 + scaling: + - operation: resize + node_group: worker-tt-dn + size: 1 + - operation: resize + node_group: worker-dn + size: 0 + - operation: resize + node_group: worker-tt + size: 0 + - operation: add + node_group: worker-tt + size: 1 + - operation: add + node_group: worker-dn + size: 1 + edp_jobs_flow: hadoop_1 \ No newline at end of file diff --git a/etc/scenario/sahara-ci/vanilla-2.6.0.yaml b/etc/scenario/sahara-ci/vanilla-2.6.0.yaml new file mode 100644 index 0000000000..4d6a8fa97e --- /dev/null +++ b/etc/scenario/sahara-ci/vanilla-2.6.0.yaml @@ -0,0 +1,86 @@ +clusters: + - plugin_name: vanilla + plugin_version: 2.6.0 + image: image_name_for_vanilla_2_6_0 + node_group_templates: + - name: worker-dn-nm + flavor_id: '3' + node_processes: + - datanode + - nodemanager + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + MapReduce: + yarn.app.mapreduce.am.resource.mb: 256 + yarn.app.mapreduce.am.command-opts: -Xmx256m + YARN: + yarn.scheduler.minimum-allocation-mb: 256 + yarn.scheduler.maximum-allocation-mb: 1024 + yarn.nodemanager.vmem-check-enabled: false + - name: worker-nm + flavor_id: '3' + node_processes: + - nodemanager + auto_security_group: true + node_configs: + *ng_configs + - name: worker-dn + flavor_id: '3' + node_processes: + - datanode + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + *ng_configs + - name: master-rm-nn-hvs + flavor_id: '3' + node_processes: + - namenode + - resourcemanager + - hiveserver + auto_security_group: true + node_configs: + *ng_configs + - name: master-oo-hs-sn + flavor_id: '3' + node_processes: + - oozie + - historyserver + - secondarynamenode + auto_security_group: true + node_configs: + *ng_configs + cluster_template: + name: vanilla-2-6 + node_group_templates: + master-rm-nn-hvs: 1 + master-oo-hs-sn: 1 + worker-dn-nm: 2 + worker-dn: 1 + worker-nm: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: scenario-vanilla2 + scaling: + - operation: resize + node_group: worker-dn-nm + size: 1 + - operation: resize + node_group: worker-dn + size: 0 + - operation: resize + node_group: worker-nm + size: 0 + - operation: add + node_group: worker-dn + size: 1 + - operation: add + node_group: worker-nm + size: 1 + edp_jobs_flow: hadoop_2