diff --git a/etc/scenario/sahara-ci/README.rst b/etc/scenario/sahara-ci/README.rst index 119090b5..49e4d4eb 100644 --- a/etc/scenario/sahara-ci/README.rst +++ b/etc/scenario/sahara-ci/README.rst @@ -9,18 +9,19 @@ tests. Details ------- -Key values: +Key values (mako variables): -* %OS_USERNAME%, %OS_PASSWORD%, %OS_TENANT_NAME%, %OPENSTACK_HOST% - OpenStack credentials; -* %NETWORK% - network type (neutron or nova-network); -* %cluster_name% - name of cluster, which generating from $HOST-$ZUUL_CHANGE-$CLUSTER_HASH. Where: +* ${OS_USERNAME}, ${OS_PASSWORD}, ${OS_TENANT_NAME}, ${OS_AUTH_URL} - OpenStack credentials and access details +* ${network_type} - network type (neutron or nova-network); +* ${network_private_name}, ${network_public_name} - names of private (tenant) and public networks; +* ${cluster_name} - name of cluster, which generating from $HOST-$ZUUL_CHANGE-$CLUSTER_HASH. Where: * $HOST - host id (c1 - with neutron, c2 - with nova-network); * $ZUUL_CHANGE - change number; * $CLUSTER_HASH - hash, which generating for each cluster by using "uuid" python module; -* %{plugin}_image% - name of image for each plugin; +* ${_image} - name of image for each plugin; * flavor ids: - * %ci_flavor% - 2GB RAM, 1 VCPU, 40GB Root disk; - * %medium_flavor% - 4GB RAM, 2 VCPUs, 40GB Root disk; + * ${ci_flavor} - 2GB RAM, 1 VCPU, 40GB Root disk; + * ${medium_flavor} - 4GB RAM, 2 VCPUs, 40GB Root disk; Main URLs --------- diff --git a/etc/scenario/sahara-ci/cdh-5.3.0.yaml.mako b/etc/scenario/sahara-ci/cdh-5.3.0.yaml.mako new file mode 100644 index 00000000..2abefbd8 --- /dev/null +++ b/etc/scenario/sahara-ci/cdh-5.3.0.yaml.mako @@ -0,0 +1,71 @@ +clusters: + - plugin_name: cdh + plugin_version: 5.3.0 + image: ${cdh_image} + node_group_templates: + - name: worker-dn + flavor_id: ${ci_flavor_id} + node_processes: + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + DATANODE: + dfs_datanode_du_reserved: 0 + - name: worker-nm + flavor_id: ${ci_flavor_id} + node_processes: + - YARN_NODEMANAGER + auto_security_group: true + - name: worker-nm-dn + flavor_id: ${ci_flavor_id} + node_processes: + - YARN_NODEMANAGER + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + *ng_configs + - name: manager + flavor_id: ${medium_flavor_id} + node_processes: + - CLOUDERA_MANAGER + auto_security_group: true + - name: master-core + flavor_id: ${medium_flavor_id} + node_processes: + - HDFS_NAMENODE + - YARN_RESOURCEMANAGER + - SENTRY_SERVER + - ZOOKEEPER_SERVER + auto_security_group: true + - name: master-additional + flavor_id: ${medium_flavor_id} + node_processes: + - OOZIE_SERVER + - YARN_JOBHISTORY + - HDFS_SECONDARYNAMENODE + - HIVE_METASTORE + - HIVE_SERVER2 + auto_security_group: true + cluster_template: + name: cdh530 + node_group_templates: + manager: 1 + master-core: 1 + master-additional: 1 + worker-nm-dn: 1 + worker-nm: 1 + worker-dn: 1 + cluster_configs: + HDFS: + dfs_replication: 1 + cluster: + name: ${cluster_name} + scenario: + - run_jobs + - sentry + edp_jobs_flow: hadoop_2 diff --git a/etc/scenario/sahara-ci/cdh-5.4.0.yaml.mako b/etc/scenario/sahara-ci/cdh-5.4.0.yaml.mako new file mode 100644 index 00000000..d8ebc6d0 --- /dev/null +++ b/etc/scenario/sahara-ci/cdh-5.4.0.yaml.mako @@ -0,0 +1,69 @@ +clusters: + - plugin_name: cdh + plugin_version: 5.4.0 + image: ${cdh_5_4_0_image} + node_group_templates: + - name: worker-dn + flavor_id: ${ci_flavor_id} + node_processes: + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + DATANODE: + dfs_datanode_du_reserved: 0 + - name: worker-nm + flavor_id: ${ci_flavor_id} + node_processes: + - YARN_NODEMANAGER + auto_security_group: true + - name: worker-nm-dn + flavor_id: ${ci_flavor_id} + node_processes: + - YARN_NODEMANAGER + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + *ng_configs + - name: manager + flavor_id: ${large_flavor_id} + node_processes: + - CLOUDERA_MANAGER + - KMS + auto_security_group: true + - name: master-core + flavor_id: ${medium_flavor_id} + node_processes: + - HDFS_NAMENODE + - YARN_RESOURCEMANAGER + auto_security_group: true + - name: master-additional + flavor_id: ${medium_flavor_id} + node_processes: + - OOZIE_SERVER + - YARN_JOBHISTORY + - HDFS_SECONDARYNAMENODE + - HIVE_METASTORE + - HIVE_SERVER2 + auto_security_group: true + cluster_template: + name: cdh540 + node_group_templates: + manager: 1 + master-core: 1 + master-additional: 1 + worker-nm-dn: 1 + worker-nm: 1 + worker-dn: 1 + cluster_configs: + HDFS: + dfs_replication: 1 + cluster: + name: ${cluster_name} + scenario: + - run_jobs + edp_jobs_flow: hadoop_2 diff --git a/etc/scenario/sahara-ci/credentials.yaml.mako b/etc/scenario/sahara-ci/credentials.yaml.mako new file mode 100644 index 00000000..2b790618 --- /dev/null +++ b/etc/scenario/sahara-ci/credentials.yaml.mako @@ -0,0 +1,10 @@ +credentials: + os_username: ${OS_USERNAME} + os_password: ${OS_PASSWORD} + os_tenant: ${OS_TENANT_NAME} + os_auth_url: ${OS_AUTH_URL} + +network: + type: ${network_type} + private_network: ${network_private_name} + public_network: ${network_public_name} diff --git a/etc/scenario/sahara-ci/fake.yaml.mako b/etc/scenario/sahara-ci/fake.yaml.mako new file mode 100644 index 00000000..fdd4a6dd --- /dev/null +++ b/etc/scenario/sahara-ci/fake.yaml.mako @@ -0,0 +1,31 @@ +clusters: + - plugin_name: fake + plugin_version: "0.1" + image: ${fake_plugin_image} + node_group_templates: + - name: worker + flavor_id: ${ci_flavor_id} + node_processes: + - datanode + - tasktracker + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + - name: master + flavor_id: ${ci_flavor_id} + node_processes: + - jobtracker + - namenode + auto_security_group: true + cluster_template: + name: fake01 + node_group_templates: + master: 1 + worker: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: transient diff --git a/etc/scenario/sahara-ci/hdp-1.3.2.yaml.mako b/etc/scenario/sahara-ci/hdp-1.3.2.yaml.mako new file mode 100644 index 00000000..c4a0caf0 --- /dev/null +++ b/etc/scenario/sahara-ci/hdp-1.3.2.yaml.mako @@ -0,0 +1,40 @@ +clusters: + - plugin_name: hdp + plugin_version: 1.3.2 + image: ${hdp_image} + node_group_templates: + - name: master + flavor_id: ${ci_flavor_id} + node_processes: + - JOBTRACKER + - NAMENODE + - SECONDARY_NAMENODE + - GANGLIA_SERVER + - NAGIOS_SERVER + - AMBARI_SERVER + - OOZIE_SERVER + auto_security_group: false + - name: worker + flavor_id: ${ci_flavor_id} + node_processes: + - TASKTRACKER + - DATANODE + - HDFS_CLIENT + - MAPREDUCE_CLIENT + - OOZIE_CLIENT + - PIG + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: false + cluster_template: + name: hdp132 + node_group_templates: + master: 1 + worker: 3 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: hadoop_1 diff --git a/etc/scenario/sahara-ci/hdp-2.0.6.yaml.mako b/etc/scenario/sahara-ci/hdp-2.0.6.yaml.mako new file mode 100644 index 00000000..cd209483 --- /dev/null +++ b/etc/scenario/sahara-ci/hdp-2.0.6.yaml.mako @@ -0,0 +1,47 @@ +clusters: + - plugin_name: hdp + plugin_version: 2.0.6 + image: ${hdp_two_image} + node_group_templates: + - name: master + flavor_id: ${ci_flavor_id} + node_processes: + - AMBARI_SERVER + - GANGLIA_SERVER + - HISTORYSERVER + - NAGIOS_SERVER + - NAMENODE + - OOZIE_SERVER + - RESOURCEMANAGER + - SECONDARY_NAMENODE + - ZOOKEEPER_SERVER + auto_security_group: true + - name: worker + flavor_id: ${ci_flavor_id} + node_processes: + - DATANODE + - HDFS_CLIENT + - MAPREDUCE2_CLIENT + - NODEMANAGER + - OOZIE_CLIENT + - PIG + - YARN_CLIENT + - ZOOKEEPER_CLIENT + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + cluster_template: + name: hdp206 + node_group_templates: + master: 1 + worker: 3 + cluster_configs: + YARN: + yarn.log-aggregation-enable: false + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: hadoop_2 diff --git a/etc/scenario/sahara-ci/mapr-4.0.2.mrv2.yaml.mako b/etc/scenario/sahara-ci/mapr-4.0.2.mrv2.yaml.mako new file mode 100644 index 00000000..7fea3961 --- /dev/null +++ b/etc/scenario/sahara-ci/mapr-4.0.2.mrv2.yaml.mako @@ -0,0 +1,47 @@ +clusters: + - plugin_name: mapr + plugin_version: 4.0.2.mrv2 + image: ${mapr_402mrv2_image} + node_group_templates: + - name: master + flavor_id: ${ci_flavor_id} + node_processes: + - Metrics + - Webserver + - ZooKeeper + - HTTPFS + - Oozie + - FileServer + - CLDB + - Flume + - Hue + - NodeManager + - HistoryServer + - ResourceManager + - HiveServer2 + - HiveMetastore + - Sqoop2-Client + - Sqoop2-Server + auto_security_group: true + volumes_per_node: 2 + volumes_size: 20 + - name: worker + flavor_id: ${ci_flavor_id} + node_processes: + - NodeManager + - FileServer + auto_security_group: true + volumes_per_node: 2 + volumes_size: 20 + cluster_template: + name: mapr402mrv2 + node_group_templates: + master: 1 + worker: 3 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: mapr diff --git a/etc/scenario/sahara-ci/spark-1.0.0.yaml.mako b/etc/scenario/sahara-ci/spark-1.0.0.yaml.mako new file mode 100644 index 00000000..dcf0e0c4 --- /dev/null +++ b/etc/scenario/sahara-ci/spark-1.0.0.yaml.mako @@ -0,0 +1,32 @@ +clusters: + - plugin_name: spark + plugin_version: 1.0.0 + image: ${spark_image} + node_group_templates: + - name: master + flavor_id: ${ci_flavor_id} + node_processes: + - master + - namenode + auto_security_group: true + - name: worker + flavor_id: ${ci_flavor_id} + node_processes: + - datanode + - slave + auto_security_group: true + cluster_template: + name: spark100 + node_group_templates: + master: 1 + worker: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: spark_edp diff --git a/etc/scenario/sahara-ci/transient.yaml.mako b/etc/scenario/sahara-ci/transient.yaml.mako new file mode 100644 index 00000000..7aaf876e --- /dev/null +++ b/etc/scenario/sahara-ci/transient.yaml.mako @@ -0,0 +1,50 @@ +clusters: + - plugin_name: vanilla + plugin_version: 2.6.0 + image: ${vanilla_two_six_image} + node_group_templates: + - name: worker + flavor_id: ${ci_flavor_id} + node_processes: + - datanode + - nodemanager + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + MapReduce: + yarn.app.mapreduce.am.resource.mb: 256 + yarn.app.mapreduce.am.command-opts: -Xmx256m + YARN: + yarn.scheduler.minimum-allocation-mb: 256 + yarn.scheduler.maximum-allocation-mb: 1024 + yarn.nodemanager.vmem-check-enabled: false + - name: master + flavor_id: ${ci_flavor_id} + node_processes: + - oozie + - historyserver + - resourcemanager + - namenode + auto_security_group: true + cluster_template: + name: transient + node_group_templates: + master: 1 + worker: 3 + cluster_configs: + HDFS: + dfs.replication: 1 + MapReduce: + mapreduce.tasktracker.map.tasks.maximum: 16 + mapreduce.tasktracker.reduce.tasks.maximum: 16 + YARN: + yarn.resourcemanager.scheduler.class: org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler + cluster: + name: ${cluster_name} + is_transient: true + scenario: + - run_jobs + - transient + edp_jobs_flow: transient diff --git a/etc/scenario/sahara-ci/vanilla-1.2.1.yaml.mako b/etc/scenario/sahara-ci/vanilla-1.2.1.yaml.mako new file mode 100644 index 00000000..6206741f --- /dev/null +++ b/etc/scenario/sahara-ci/vanilla-1.2.1.yaml.mako @@ -0,0 +1,73 @@ +clusters: + - plugin_name: vanilla + plugin_version: 1.2.1 + image: ${vanilla_image} + node_group_templates: + - name: worker-tt-dn + flavor_id: ${ci_flavor_id} + node_processes: + - datanode + - tasktracker + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + - name: worker-tt + flavor_id: ${ci_flavor_id} + node_processes: + - tasktracker + auto_security_group: true + - name: worker-dn + flavor_id: ${ci_flavor_id} + node_processes: + - datanode + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + - name: master-jt-nn + flavor_id: ${ci_flavor_id} + node_processes: + - namenode + - jobtracker + auto_security_group: true + - name: master-sec-nn-oz + flavor_id: ${ci_flavor_id} + node_processes: + - oozie + - secondarynamenode + auto_security_group: true + + cluster_template: + name: vanilla121 + node_group_templates: + master-sec-nn-oz: 1 + master-jt-nn: 1 + worker-tt: 1 + worker-tt-dn: 2 + worker-dn: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + MapReduce: + mapred.map.tasks.speculative.execution: False + mapred.child.java.opts: -Xmx512m + general: + 'Enable Swift': True + cluster: + name: ${cluster_name} + scaling: + - operation: resize + node_group: worker-tt-dn + size: 1 + - operation: resize + node_group: worker-dn + size: 0 + - operation: resize + node_group: worker-tt + size: 0 + - operation: add + node_group: worker-tt + size: 1 + - operation: add + node_group: worker-dn + size: 1 + edp_jobs_flow: hadoop_1 diff --git a/etc/scenario/sahara-ci/vanilla-2.6.0.yaml.mako b/etc/scenario/sahara-ci/vanilla-2.6.0.yaml.mako new file mode 100644 index 00000000..c79da060 --- /dev/null +++ b/etc/scenario/sahara-ci/vanilla-2.6.0.yaml.mako @@ -0,0 +1,86 @@ +clusters: + - plugin_name: vanilla + plugin_version: 2.6.0 + image: ${vanilla_two_six_image} + node_group_templates: + - name: worker-dn-nm + flavor_id: ${ci_flavor_id} + node_processes: + - datanode + - nodemanager + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + MapReduce: + yarn.app.mapreduce.am.resource.mb: 256 + yarn.app.mapreduce.am.command-opts: -Xmx256m + YARN: + yarn.scheduler.minimum-allocation-mb: 256 + yarn.scheduler.maximum-allocation-mb: 1024 + yarn.nodemanager.vmem-check-enabled: false + - name: worker-nm + flavor_id: ${ci_flavor_id} + node_processes: + - nodemanager + auto_security_group: true + node_configs: + *ng_configs + - name: worker-dn + flavor_id: ${ci_flavor_id} + node_processes: + - datanode + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + *ng_configs + - name: master-rm-nn-hvs + flavor_id: ${ci_flavor_id} + node_processes: + - namenode + - resourcemanager + - hiveserver + auto_security_group: true + node_configs: + *ng_configs + - name: master-oo-hs-sn + flavor_id: ${ci_flavor_id} + node_processes: + - oozie + - historyserver + - secondarynamenode + auto_security_group: true + node_configs: + *ng_configs + cluster_template: + name: vanilla260 + node_group_templates: + master-rm-nn-hvs: 1 + master-oo-hs-sn: 1 + worker-dn-nm: 2 + worker-dn: 1 + worker-nm: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: resize + node_group: worker-dn-nm + size: 1 + - operation: resize + node_group: worker-dn + size: 0 + - operation: resize + node_group: worker-nm + size: 0 + - operation: add + node_group: worker-dn + size: 1 + - operation: add + node_group: worker-nm + size: 1 + edp_jobs_flow: hadoop_2