openstack/deb-sahara
Code Issues Proposed changes

Merge "Add scenario files for new integration tests"

Browse Source
This commit is contained in:
Jenkins 2015-03-23 11:41:59 +00:00 committed by Gerrit Code Review
commit 352aa655a6
10 changed files with 507 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
etc/scenario/sahara-ci/README.rst Normal file
View File

@ -0,0 +1,5 @@
=========
Sahara CI
=========
This files need for sahara-ci. Modifying these files change the behavior tests

76
etc/scenario/sahara-ci/cdh-5.3.0.yaml Normal file
View File

@ -0,0 +1,76 @@
clusters:
- plugin_name: cdh
plugin_version: 5.3.0
image: ubuntu_sahara_cloudera
node_group_templates:
- name: worker-dn
flavor_id: '3'
node_processes:
- HDFS_DATANODE
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
node_configs:
&ng_configs
DATANODE:
dfs_datanode_du_reserved: 0
- name: worker-nm
flavor_id: '3'
node_processes:
- YARN_NODEMANAGER
auto_security_group: true
- name: worker-nm-dn
flavor_id: '3'
node_processes:
- YARN_NODEMANAGER
- HDFS_DATANODE
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
node_configs:
*ng_configs
- name: manager
flavor_id: '3'
node_processes:
- CLOUDERA_MANAGER
auto_security_group: true
- name: master-core
flavor_id: '3'
node_processes:
- HDFS_NAMENODE
- YARN_RESOURCEMANAGER
auto_security_group: true
- name: master-additional
flavor_id: '3'
node_processes:
- OOZIE_SERVER
- YARN_JOBHISTORY
- HDFS_SECONDARYNAMENODE
- HIVE_METASTORE
- HIVE_SERVER2
auto_security_group: true
cluster_template:
name: cdh-5-3-0
node_group_templates:
manager: 1
master-core: 1
master-additional: 1
worker-nm-dn: 1
worker-nm: 1
worker-dn: 1
cluster_configs:
HDFS:
dfs_replication: 1
cluster:
name: scenario-cdh
scaling:
- operation: add
node_group: worker-nm-dn
size: 1
- operation: resize
node_group: worker-nm
size: 0
- operation: resize
node_group: worker-dn
size: 0
edp_jobs_flow: hadoop_2

9
etc/scenario/sahara-ci/credentials.yaml Normal file
View File

@ -0,0 +1,9 @@
credentials:
os_username: username
os_password: password
os_tenant: tenant
os_auth_url: http://127.0.0.1:5000/v2.0
network:
private_network: private
public_network: public

94
etc/scenario/sahara-ci/edp.yaml Normal file
View File

@ -0,0 +1,94 @@
edp_jobs_flow:
hadoop_2:
- type: Pig
input_datasource:
type: swift
source: etc/edp-examples/edp-pig/top-todoers/data/input
output_datasource:
type: hdfs
destination: /user/hadoop/edp-output
main_lib:
type: swift
source: etc/edp-examples/edp-pig/top-todoers/example.pig
- type: MapReduce
input_datasource:
type: swift
source: etc/edp-examples/edp-pig/trim-spaces/data/input
output_datasource:
type: hdfs
destination: /user/hadoop/edp-output
additional_libs:
- type: database
source: etc/edp-examples/edp-mapreduce/edp-mapreduce.jar
configs:
mapred.mapper.class: org.apache.oozie.example.SampleMapper
mapred.reducer.class: org.apache.oozie.example.SampleReducer
- type: MapReduce.Streaming
input_datasource:
type: swift
source: etc/edp-examples/edp-pig/trim-spaces/data/input
output_datasource:
type: hdfs
destination: /user/hadoop/edp-output
configs:
edp.streaming.mapper: /bin/cat
edp.streaming.reducer: /usr/bin/wc
- type: Java
additional_libs:
- type: database
source: etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.6.0.jar
configs:
edp.java.main_class: org.apache.hadoop.examples.QuasiMonteCarlo
args:
- 10
- 10
hadoop_1:
- type: Pig
input_datasource:
type: swift
source: etc/edp-examples/edp-pig/top-todoers/data/input
output_datasource:
type: hdfs
destination: /user/hadoop/edp-output
main_lib:
type: swift
source: etc/edp-examples/edp-pig/top-todoers/example.pig
- type: MapReduce
input_datasource:
type: swift
source: etc/edp-examples/edp-pig/trim-spaces/data/input
output_datasource:
type: hdfs
destination: /user/hadoop/edp-output
additional_libs:
- type: database
source: etc/edp-examples/edp-mapreduce/edp-mapreduce.jar
configs:
dfs.replication: 1
mapred.mapper.class: org.apache.oozie.example.SampleMapper
mapred.reducer.class: org.apache.oozie.example.SampleReducer
- type: MapReduce.Streaming
input_datasource:
type: swift
source: etc/edp-examples/edp-pig/trim-spaces/data/input
output_datasource:
type: hdfs
destination: /user/hadoop/edp-output
configs:
edp.streaming.mapper: /bin/cat
edp.streaming.reducer: /usr/bin/wc
- type: Java
additional_libs:
- type: database
source: etc/edp-examples/hadoop2/edp-java/edp-java.jar
configs:
edp.java.main_class: org.openstack.sahara.examples.WordCount
spark_edp:
- type: Spark
main_lib:
type: database
source: etc/edp-examples/edp-spark/spark-example.jar
configs:
edp.java.main_class: org.apache.spark.examples.SparkPi
args:
- 4

46
etc/scenario/sahara-ci/hdp-1.3.2.yaml Normal file
View File

@ -0,0 +1,46 @@
clusters:
- plugin_name: hdp
plugin_version: 1.3.2
image: image_name_for_hdp_1_3_2
node_group_templates:
- name: master
flavor_id: '3'
node_processes:
- JOBTRACKER
- NAMENODE
- SECONDARY_NAMENODE
- GANGLIA_SERVER
- NAGIOS_SERVER
- AMBARI_SERVER
- OOZIE_SERVER
auto_security_group: true
- name: worker
flavor_id: '3'
node_processes:
- TASKTRACKER
- DATANODE
- HDFS_CLIENT
- MAPREDUCE_CLIENT
- OOZIE_CLIENT
- PIG
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
node_configs:
&ng_configs
DATANODE:
dfs_datanode_du_reserved: 0
cluster_template:
name: hdp-1-3-2
node_group_templates:
master: 1
worker: 3
HDFS:
datanode_du_reserved: 0
cluster:
name: scenario-hdp1
scaling:
- operation: add
node_group: worker
size: 1
edp_jobs_flow: hadoop_1

44
etc/scenario/sahara-ci/hdp-2.0.6.yaml Normal file
View File

@ -0,0 +1,44 @@
clusters:
- plugin_name: hdp
plugin_version: 2.0.6
image: image_name_for_hdp_2_0_6
node_group_templates:
- name: master
flavor_id: '3'
node_processes:
- AMBARI_SERVER
- GANGLIA_SERVER
- HISTORYSERVER
- NAGIOS_SERVER
- NAMENODE
- OOZIE_SERVER
- RESOURCEMANAGER
- SECONDARY_NAMENODE
- ZOOKEEPER_SERVER
auto_security_group: true
- name: worker
flavor_id: '3'
node_processes:
- DATANODE
- HDFS_CLIENT
- MAPREDUCE2_CLIENT
- NODEMANAGER
- OOZIE_CLIENT
- PIG
- YARN_CLIENT
- ZOOKEEPER_CLIENT
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
cluster_template:
name: hdp-2-0-6
node_group_templates:
master: 1
worker: 3
cluster_configs:
YARN:
yarn.log-aggregation-enable: false
HDFS:
datanode_du_reserved: 0
cluster:
name: scenario-hdp2

32
etc/scenario/sahara-ci/spark-1.0.0.yaml Normal file
View File

@ -0,0 +1,32 @@
clusters:
- plugin_name: spark
plugin_version: 1.0.0
image: image_name_for_spark_1_0_0
node_group_templates:
- name: master
flavor_id: '3'
node_processes:
- master
- namenode
auto_security_group: true
- name: worker
flavor_id: '3'
node_processes:
- datanode
- slave
auto_security_group: true
cluster_template:
name: spark-1-0-0
node_group_templates:
master: 1
worker: 1
cluster_configs:
HDFS:
dfs.replication: 1
cluster:
name: scenario-spark
scaling:
- operation: add
node_group: worker
size: 1
edp_jobs_flow: spark_edp

42
etc/scenario/sahara-ci/transient.yaml Normal file
View File

@ -0,0 +1,42 @@
clusters:
- plugin_name: vanilla
plugin_version: 2.6.0
image: image_name_for_vanilla_2_6_0
node_group_templates:
- name: transient_ng
flavor_id: '3'
node_processes:
- datanode
- nodemanager
- oozie
- historyserver
- secondarynamenode
- resourcemanager
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
node_configs:
&ng_configs
MapReduce:
yarn.app.mapreduce.am.resource.mb: 256
yarn.app.mapreduce.am.command-opts: -Xmx256m
YARN:
yarn.scheduler.minimum-allocation-mb: 256
yarn.scheduler.maximum-allocation-mb: 1024
yarn.nodemanager.vmem-check-enabled: false
cluster_template:
name: vanilla-2-6
node_group_templates:
transient_ng: 1
cluster_configs:
HDFS:
dfs.replication: 1
MapReduce:
mapreduce.tasktracker.map.tasks.maximum: 16,
mapreduce.tasktracker.reduce.tasks.maximum: 16
YARN:
yarn.resourcemanager.scheduler.class: org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
cluster:
name: scenario-transient
is_transient: true
edp_jobs_flow: hadoop_2

73
etc/scenario/sahara-ci/vanilla-1.2.1.yaml Normal file
View File

@ -0,0 +1,73 @@
clusters:
- plugin_name: vanilla
plugin_version: 1.2.1
image: image_name_for_vanilla_1_2_1
node_group_templates:
- name: worker-tt-dn
flavor_id: '3'
node_processes:
- datanode
- tasktracker
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
- name: worker-tt
flavor_id: '3'
node_processes:
- tasktracker
auto_security_group: true
- name: worker-dn
flavor_id: '3'
node_processes:
- datanode
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
- name: master-jt-nn
flavor_id: '3'
node_processes:
- namenode
- jobtracker
auto_security_group: true
- name: master-sec-nn-oz
flavor_id: '3'
node_processes:
- oozie
- secondarynamenode
auto_security_group: true
cluster_template:
name: vanilla-1-2-1
node_group_templates:
master-sec-nn-oz: 1
master-jt-nn: 1
worker-tt: 1
worker-tt-dn: 2
worker-dn: 1
cluster_configs:
HDFS:
dfs.replication: 1
MapReduce:
mapred.map.tasks.speculative.execution: False
mapred.child.java.opts: -Xmx500m
general:
'Enable Swift': True
cluster:
name: scenario-vanilla1
scaling:
- operation: resize
node_group: worker-tt-dn
size: 1
- operation: resize
node_group: worker-dn
size: 0
- operation: resize
node_group: worker-tt
size: 0
- operation: add
node_group: worker-tt
size: 1
- operation: add
node_group: worker-dn
size: 1
edp_jobs_flow: hadoop_1

86
etc/scenario/sahara-ci/vanilla-2.6.0.yaml Normal file
View File

@ -0,0 +1,86 @@
clusters:
- plugin_name: vanilla
plugin_version: 2.6.0
image: image_name_for_vanilla_2_6_0
node_group_templates:
- name: worker-dn-nm
flavor_id: '3'
node_processes:
- datanode
- nodemanager
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
node_configs:
&ng_configs
MapReduce:
yarn.app.mapreduce.am.resource.mb: 256
yarn.app.mapreduce.am.command-opts: -Xmx256m
YARN:
yarn.scheduler.minimum-allocation-mb: 256
yarn.scheduler.maximum-allocation-mb: 1024
yarn.nodemanager.vmem-check-enabled: false
- name: worker-nm
flavor_id: '3'
node_processes:
- nodemanager
auto_security_group: true
node_configs:
*ng_configs
- name: worker-dn
flavor_id: '3'
node_processes:
- datanode
volumes_per_node: 2
volumes_size: 2
auto_security_group: true
node_configs:
*ng_configs
- name: master-rm-nn-hvs
flavor_id: '3'
node_processes:
- namenode
- resourcemanager
- hiveserver
auto_security_group: true
node_configs:
*ng_configs
- name: master-oo-hs-sn
flavor_id: '3'
node_processes:
- oozie
- historyserver
- secondarynamenode
auto_security_group: true
node_configs:
*ng_configs
cluster_template:
name: vanilla-2-6
node_group_templates:
master-rm-nn-hvs: 1
master-oo-hs-sn: 1
worker-dn-nm: 2
worker-dn: 1
worker-nm: 1
cluster_configs:
HDFS:
dfs.replication: 1
cluster:
name: scenario-vanilla2
scaling:
- operation: resize
node_group: worker-dn-nm
size: 1
- operation: resize
node_group: worker-dn
size: 0
- operation: resize
node_group: worker-nm
size: 0
- operation: add
node_group: worker-dn
size: 1
- operation: add
node_group: worker-nm
size: 1
edp_jobs_flow: hadoop_2