From ee1928466716c58ea0c5d73152cfb70cb62dcbbd Mon Sep 17 00:00:00 2001 From: Evgeny Sikachev Date: Wed, 5 Apr 2017 12:55:59 +0400 Subject: [PATCH] Adding of spark 2.1.0 on sahara-ci Spark 2.1.0 was added to jobs, jobs to zuul, in scripts Change-Id: Ief586729d7c9de82febb0f196f26f54c0009af01 --- config/devstack/local.sh | 2 ++ config/devstack/pre_unstack.sh | 3 ++- config/zuul/layout.yaml | 13 +++++++++++++ jenkins_job_builder/diskimage.yaml | 3 ++- jenkins_job_builder/sahara.yaml | 1 + slave-scripts/dib.sh | 6 ++++++ slave-scripts/functions-dib.sh | 3 +++ 7 files changed, 29 insertions(+), 2 deletions(-) diff --git a/config/devstack/local.sh b/config/devstack/local.sh index 1218f1b0..7556b3e2 100755 --- a/config/devstack/local.sh +++ b/config/devstack/local.sh @@ -35,6 +35,7 @@ CENTOS7_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_c7.qcow2 UBUNTU_CDH_5_9_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.9.0_u14.qcow2 CENTOS7_CDH_5_9_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.9.0_c7.qcow2 SPARK_1_6_0_IMAGE_PATH=/home/ubuntu/images/spark_1.6.0_u14.qcow2 +SPARK_2_1_0_IMAGE_PATH=/home/ubuntu/images/spark_2.1.0_u14.qcow2 MAPR_5_1_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.1.0.mrv2_u14.qcow2 MAPR_5_2_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.2.0.mrv2_u14.qcow2 STORM_1_0_1_IMAGE_PATH=/home/ubuntu/images/storm_1.0.1_u14.qcow2 @@ -94,6 +95,7 @@ openstack image create $(basename -s .qcow2 $UBUNTU_CDH_5_7_0_IMAGE_PATH) --file openstack image create $(basename -s .qcow2 $CENTOS7_CDH_5_9_0_IMAGE_PATH) --file $CENTOS7_CDH_5_9_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.9.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="centos" openstack image create $(basename -s .qcow2 $UBUNTU_CDH_5_9_0_IMAGE_PATH) --file $UBUNTU_CDH_5_9_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.9.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $SPARK_1_6_0_IMAGE_PATH) --file $SPARK_1_6_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.6.0'='True' --property '_sahara_username'="ubuntu" +openstack image create $(basename -s .qcow2 $SPARK_2_1_0_IMAGE_PATH) --file $SPARK_2_1_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_2.1.0'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $MAPR_5_1_0_MRV2_IMAGE_PATH) --file $MAPR_5_1_0_MRV2_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_5.1.0.mrv2'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $MAPR_5_2_0_MRV2_IMAGE_PATH) --file $MAPR_5_2_0_MRV2_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_5.2.0.mrv2'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $STORM_1_0_1_IMAGE_PATH) --file $STORM_1_0_1_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_storm'='True' --property '_sahara_tag_1.0.1'='True' --property '_sahara_username'="ubuntu" diff --git a/config/devstack/pre_unstack.sh b/config/devstack/pre_unstack.sh index 60348766..687ea5c2 100755 --- a/config/devstack/pre_unstack.sh +++ b/config/devstack/pre_unstack.sh @@ -27,6 +27,7 @@ CENTOS7_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_c7.qcow2 UBUNTU_CDH_5_9_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.9.0_u14.qcow2 CENTOS7_CDH_5_9_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.9.0_c7.qcow2 SPARK_1_6_0_IMAGE_PATH=/home/ubuntu/images/spark_1.6.0_u14.qcow2 +SPARK_2_1_0_IMAGE_PATH=/home/ubuntu/images/spark_2.1.0_u14.qcow2 MAPR_5_1_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.1.0.mrv2_u14.qcow2 MAPR_5_2_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.2.0.mrv2_u14.qcow2 STORM_1_0_1_IMAGE_PATH=/home/ubuntu/images/storm_1.0.1_u14.qcow2 @@ -43,9 +44,9 @@ openstack image save --file $UBUNTU_CDH_5_7_0_IMAGE_PATH $(get_id $UBUNTU_CDH_5_ openstack image save --file $CENTOS7_CDH_5_9_0_IMAGE_PATH $(get_id $CENTOS7_CDH_5_9_0_IMAGE_PATH) openstack image save --file $UBUNTU_CDH_5_9_0_IMAGE_PATH $(get_id $UBUNTU_CDH_5_9_0_IMAGE_PATH) openstack image save --file $SPARK_1_6_0_IMAGE_PATH $(get_id $SPARK_1_6_0_IMAGE_PATH) +openstack image save --file $SPARK_2_1_0_IMAGE_PATH $(get_id $SPARK_2_1_0_IMAGE_PATH) openstack image save --file $MAPR_5_1_0_MRV2_IMAGE_PATH $(get_id $MAPR_5_1_0_MRV2_IMAGE_PATH) openstack image save --file $MAPR_5_2_0_MRV2_IMAGE_PATH $(get_id $MAPR_5_2_0_MRV2_IMAGE_PATH) openstack image save --file $STORM_1_0_1_IMAGE_PATH $(get_id $STORM_1_0_1_IMAGE_PATH) bash $TOP_DIR/unstack.sh - diff --git a/config/zuul/layout.yaml b/config/zuul/layout.yaml index 6abeaf5c..a2862711 100644 --- a/config/zuul/layout.yaml +++ b/config/zuul/layout.yaml @@ -72,6 +72,7 @@ projects: - gate-sahara-vanilla_2.7.1-u14 - gate-sahara-spark_1.6.0-u14 - gate-sahara-spark_1.6.0-u14-python3 + - gate-sahara-spark_2.1.0-u14 - gate-sahara-mapr_5.1.0.mrv2-u14 - gate-sahara-mapr_5.2.0.mrv2-u14 - gate-sahara-cdh_5.5.0-u14 @@ -90,6 +91,7 @@ projects: check: - gate-sahara_tests-vanilla_2.7.1-u14 - gate-sahara_tests-spark_1.6.0-u14 + - gate-sahara_tests-spark_2.1.0-u14 - gate-sahara_tests-mapr_5.1.0.mrv2-u14 - gate-sahara_tests-mapr_5.2.0.mrv2-u14 - gate-sahara_tests-cdh_5.5.0-u14 @@ -204,6 +206,17 @@ jobs: - ^sahara_tests/.*.py$ - ^diskimage-create/.*$ - ^elements/.*$ + - name: ^(?!nightly).*-spark_2.1.0-.*$ + skip-if: + - branch: ^(stable/newton|stable/mitaka)$ + files: + - ^sahara_tests/scenario/defaults/spark-2.1.0.*$ + - ^sahara_tests/scenario/defaults/edp.yaml.mako$ + - ^sahara_tests/scenario/defaults/edp-examples/.*$ + - ^sahara/.*$ + - ^sahara_tests/.*.py$ + - ^diskimage-create/.*$ + - ^elements/.*$ - name: ^(?!nightly).*-cdh_5.5.0-.*$ skip-if: - branch: ^(stable/newton|master)$ diff --git a/jenkins_job_builder/diskimage.yaml b/jenkins_job_builder/diskimage.yaml index 42d02b1b..aa8343bd 100644 --- a/jenkins_job_builder/diskimage.yaml +++ b/jenkins_job_builder/diskimage.yaml @@ -16,7 +16,7 @@ - project: name: sahara-image-elements github-org: openstack - plugin: + plugin: - vanilla_2.7.1 - cdh_5.5.0 - cdh_5.7.0 @@ -30,6 +30,7 @@ - 'dib-{plugin}-{os}': plugin: - spark_1.6.0 + - spark_2.1.0 - storm_1.0.1 - mapr_5.1.0.mrv2 - mapr_5.2.0.mrv2 diff --git a/jenkins_job_builder/sahara.yaml b/jenkins_job_builder/sahara.yaml index e34ebdda..46c82ce7 100644 --- a/jenkins_job_builder/sahara.yaml +++ b/jenkins_job_builder/sahara.yaml @@ -47,6 +47,7 @@ - mapr_5.1.0.mrv2 - mapr_5.2.0.mrv2 - spark_1.6.0 + - spark_2.1.0 - storm_1.0.1 os: u14 - '{pipeline}-{project_name}-{plugin}-{os}': diff --git a/slave-scripts/dib.sh b/slave-scripts/dib.sh index dc183837..49602bcd 100755 --- a/slave-scripts/dib.sh +++ b/slave-scripts/dib.sh @@ -66,6 +66,12 @@ case $plugin in upload_image "${plugin}" "${username}" "${spark_1_6_0_image:?}" ;; + spark_2.1.0) + env ubuntu_spark_image_name="${spark_2_1_0_image:?}" SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p spark -s 2.1.0 + check_error_code $? "${spark_2_1_0_image:?}".qcow2 + upload_image "${plugin}" "${username}" "${spark_2_1_0_image:?}" + ;; + cdh_5.5.0) env cloudera_5_5_${os_type}_image_name="${cdh_5_5_0_image:?}" SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i $os_type -v 5.5 check_error_code $? "${cdh_5_5_0_image:?}".qcow2 diff --git a/slave-scripts/functions-dib.sh b/slave-scripts/functions-dib.sh index 7442970f..0151fbf4 100755 --- a/slave-scripts/functions-dib.sh +++ b/slave-scripts/functions-dib.sh @@ -74,6 +74,9 @@ upload_image() { spark_1.6.0) image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.6.0=True --property _sahara_username=${username}" ;; + spark_2.1.0) + image_properties="--property _sahara_tag_spark=True --property _sahara_tag_2.1.0=True --property _sahara_username=${username}" + ;; mapr_5.1.0.mrv2) image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_5.1.0.mrv2=True --property _sahara_username=${username}" ;;