#!/bin/bash -xe # source CI credentials . /home/jenkins/ci_openrc # source functions . $FUNCTION_PATH/functions-common.sh . $FUNCTION_PATH/functions-dib.sh CLUSTER_HASH=${CLUSTER_HASH:-$RANDOM} cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH" SAHARA_PATH="/tmp/sahara" # default (deprecated) config file for integration tests tests_config_file="$SAHARA_PATH/sahara/tests/integration/configs/itest.conf" tests_config_file_template="$sahara_templates_configs_path/itest.conf.sample" sahara_conf_path="$SAHARA_PATH/etc/sahara/sahara.conf" engine=$(echo $JOB_NAME | awk -F '-' '{ print $3 }') job_type="$1" image_type=${2:-ubuntu} # Image names vanilla_image=$HOST-sahara-vanilla-${image_type}-${ZUUL_CHANGE}-hadoop_1 vanilla_two_four_image=$HOST-sahara-vanilla-${image_type}-${ZUUL_CHANGE}-hadoop_2.4 vanilla_two_six_image=$HOST-sahara-vanilla-${image_type}-${ZUUL_CHANGE}-hadoop_2.6 hdp_image=$HOST-sahara-hdp-centos-${ZUUL_CHANGE}-hadoop_1 hdp_two_image=$HOST-sahara-hdp-centos-${ZUUL_CHANGE}-hadoop_2 spark_image=$HOST-sahara-spark-ubuntu-${ZUUL_CHANGE} cdh_image=$HOST-${image_type}-cdh-${ZUUL_CHANGE} # Clone Sahara git clone https://review.openstack.org/openstack/sahara $SAHARA_PATH case $job_type in vanilla*) # Up local HTTPServer with Java source pushd /home/jenkins python -m SimpleHTTPServer 8000 > /dev/null & popd if [ "${image_type}" == 'centos' ]; then username='cloud-user' else username=${image_type} fi hadoop_version=$(echo $job_type | awk -F '_' '{print $2}') case $hadoop_version in 1) sudo DIB_REPO_PATH="/home/jenkins/diskimage-builder" ${image_type}_vanilla_hadoop_1_image_name=${vanilla_image} JAVA_DOWNLOAD_URL='http://127.0.0.1:8000/jdk-7u51-linux-x64.tar.gz' SIM_REPO_PATH=$WORKSPACE bash -x diskimage-create/diskimage-create.sh -p vanilla -i $image_type -v 1 check_error_code $? ${vanilla_image}.qcow2 upload_image "vanilla-1" "${username}" ${vanilla_image} insert_config_value $tests_config_file_template VANILLA SKIP_CINDER_TEST True insert_config_value $tests_config_file_template VANILLA SKIP_CLUSTER_CONFIG_TEST True insert_config_value $tests_config_file_template VANILLA SKIP_SCALING_TEST True insert_config_value $tests_config_file_template VANILLA IMAGE_NAME $vanilla_image plugin=vanilla1 ;; 2.4) sudo DIB_REPO_PATH="/home/jenkins/diskimage-builder" ${image_type}_vanilla_hadoop_2_4_image_name=${vanilla_two_four_image} JAVA_DOWNLOAD_URL='http://127.0.0.1:8000/jdk-7u51-linux-x64.tar.gz' SIM_REPO_PATH=$WORKSPACE bash -x diskimage-create/diskimage-create.sh -p vanilla -i $image_type -v 2.4 check_error_code $? ${vanilla_two_four_image}.qcow2 upload_image "vanilla-2.4" "${username}" ${vanilla_two_four_image} DISTRIBUTE_MODE=True insert_config_value $tests_config_file_template VANILLA_TWO SKIP_CINDER_TEST True insert_config_value $tests_config_file_template VANILLA_TWO SKIP_CLUSTER_CONFIG_TEST True insert_config_value $tests_config_file_template VANILLA_TWO SKIP_SCALING_TEST True insert_config_value $tests_config_file_template VANILLA_TWO IMAGE_NAME $vanilla_two_four_image plugin=vanilla2 ;; 2.6) sudo DIB_REPO_PATH="/home/jenkins/diskimage-builder" ${image_type}_vanilla_hadoop_2_6_image_name=${vanilla_two_six_image} JAVA_DOWNLOAD_URL='http://127.0.0.1:8000/jdk-7u51-linux-x64.tar.gz' SIM_REPO_PATH=$WORKSPACE bash -x diskimage-create/diskimage-create.sh -p vanilla -i $image_type -v 2.6 check_error_code $? ${vanilla_two_six_image}.qcow2 upload_image "vanilla-2.6" "${username}" ${vanilla_two_six_image} DISTRIBUTE_MODE=True tests_config_file="$sahara_templates_configs_path/sahara-test-config-vanilla-2.6.yaml" insert_scenario_value $tests_config_file vanilla_two_six_image ;; esac ;; spark) # Up local HTTPServer with Java source pushd /home/jenkins python -m SimpleHTTPServer 8000 > /dev/null & popd sudo DIB_REPO_PATH="/home/jenkins/diskimage-builder" ubuntu_spark_image_name=${spark_image} JAVA_DOWNLOAD_URL='http://127.0.0.1:8000/jdk-7u51-linux-x64.tar.gz' SIM_REPO_PATH=$WORKSPACE bash -x diskimage-create/diskimage-create.sh -p "spark" check_error_code $? ${spark_image}.qcow2 upload_image "spark" "ubuntu" ${spark_image} if [[ "$JOB_NAME" =~ scenario ]]; then tests_config_file="$sahara_templates_configs_path/sahara-test-config-spark.yaml" insert_scenario_value $tests_config_file spark_image else insert_config_value $tests_config_file_template SPARK SKIP_CINDER_TEST True insert_config_value $tests_config_file_template SPARK SKIP_CLUSTER_CONFIG_TEST True insert_config_value $tests_config_file_template SPARK SKIP_SCALING_TEST True insert_config_value $tests_config_file_template SPARK IMAGE_NAME $spark_image plugin=spark fi insert_config_value $sahara_conf_path DEFAULT plugins spark ;; hdp_1) sudo DIB_REPO_PATH="/home/jenkins/diskimage-builder" centos_hdp_hadoop_1_image_name=${hdp_image} SIM_REPO_PATH=$WORKSPACE bash -x diskimage-create/diskimage-create.sh -p hdp -v 1 check_error_code $? ${hdp_image}.qcow2 upload_image "hdp1" "root" ${hdp_image} insert_config_value $tests_config_file_template HDP SKIP_CINDER_TEST True insert_config_value $tests_config_file_template HDP SKIP_CLUSTER_CONFIG_TEST True insert_config_value $tests_config_file_template HDP SKIP_SCALING_TEST True insert_config_value $tests_config_file_template HDP IMAGE_NAME $hdp_image plugin=hdp1 ;; hdp_2) sudo DIB_REPO_PATH="/home/jenkins/diskimage-builder" centos_hdp_hadoop_2_image_name=${hdp_two_image} SIM_REPO_PATH=$WORKSPACE bash -x diskimage-create/diskimage-create.sh -p hdp -v 2 check_error_code $? ${hdp_two_image}.qcow2 upload_image "hdp2" "root" ${hdp_two_image} DISTRIBUTE_MODE=True insert_config_value $tests_config_file_template HDP2 SKIP_CINDER_TEST True insert_config_value $tests_config_file_template HDP2 SKIP_CLUSTER_CONFIG_TEST True insert_config_value $tests_config_file_template HDP2 SKIP_SCALING_TEST True insert_config_value $tests_config_file_template HDP2 IMAGE_NAME $hdp_two_image plugin=hdp2 ;; cdh) if [ "${image_type}" == 'centos' ]; then username='cloud-user' else username='ubuntu' fi sudo DIB_REPO_PATH="/home/jenkins/diskimage-builder" cloudera_5_3_${image_type}_image_name=${cdh_image} SIM_REPO_PATH=$WORKSPACE bash -x diskimage-create/diskimage-create.sh -p cloudera -i $image_type -v 5.3 check_error_code $? ${cdh_image}.qcow2 upload_image "cdh" ${username} ${cdh_image} insert_config_value $sahara_conf_path DEFAULT plugins cdh insert_config_value $tests_config_file_template CDH SKIP_CINDER_TEST True insert_config_value $tests_config_file_template CDH SKIP_CLUSTER_CONFIG_TEST True insert_config_value $tests_config_file_template CDH SKIP_SCALING_TEST True insert_config_value $tests_config_file_template CDH IMAGE_NAME $cdh_image plugin=cdh ;; esac cd $SAHARA_PATH if [ "$ZUUL_BRANCH" != "master" ]; then git checkout "$ZUUL_BRANCH" sudo pip install -U -r requirements.txt fi sudo pip install . enable_pypi write_sahara_main_conf "$sahara_conf_path" "$engine" write_tests_conf "$tests_config_file" "$cluster_name" start_sahara "$sahara_conf_path" && run_tests "$tests_config_file" "$plugin" print_python_env cleanup_image "$job_type" "$image_type"