#!/bin/bash set -e export IMAGE_SIZE=$DIB_IMAGE_SIZE # This will unset parameter DIB_IMAGE_SIZE for Ubuntu and Fedora vanilla images unset DIB_IMAGE_SIZE # default debug setting should be false IMAGE_GENERATION_DEBUG_MODE="false" # The default tag to use for the DIB repo DEFAULT_DIB_REPO_BRANCH="0.1.17" while getopts "p:i:v:d:m" opt; do case $opt in p) PLUGIN=$OPTARG ;; i) IMAGE_TYPE=$OPTARG ;; v) HADOOP_VERSION=$OPTARG ;; d) IMAGE_GENERATION_DEBUG_MODE=$OPTARG ;; m) if [ -n "$DIB_REPO_BRANCH" ]; then echo "Error: DIB_REPO_BRANCH set and -m requested, please choose one." exit 3 else DIB_REPO_BRANCH="master" fi ;; *) echo echo "Usage: $(basename $0) [-p vanilla|spark|hdp|idh] [-i ubuntu|fedora|centos] [-v 1|2|plain] [-d true|false] [-m]" echo "'-p' is plugin version, '-i' is image type, '-v' is hadoop version, '-d controls the debug mode for image generation (false by default)" echo "'-m' set the diskimage-builder repo to the master branch (default: $DEFAULT_DIB_REPO_BRANCH)" echo "You shouldn't specify hadoop version and image type for spark plugin" echo "You shouldn't specify image type for hdp plugin" echo "Version 'plain' could be specified for hdp plugin only" echo "Debug mode should only be enabled for local debugging purposes, not for production systems" echo "By default all images for all plugins will be created" exit 1 ;; esac done if [ -z $DIB_REPO_BRANCH ]; then DIB_REPO_BRANCH=$DEFAULT_DIB_REPO_BRANCH fi # Checks of input if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "idh" ]; then echo -e "Unknown plugin selected.\nAborting" exit 1 fi if [ -n "$IMAGE_TYPE" -a "$IMAGE_TYPE" != "ubuntu" -a "$IMAGE_TYPE" != "fedora" -a "$IMAGE_TYPE" != "centos" ]; then echo -e "Unknown image type selected.\nAborting" exit 1 fi if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then echo -e "Unknown hadoop version selected.\nAborting" exit 1 fi if [ -n "$IMAGE_GENERATION_DEBUG_MODE" -a "$IMAGE_GENERATION_DEBUG_MODE" != "true" -a "$IMAGE_GENERATION_DEBUG_MODE" != "false" ]; then echo -e "Unknown debug mode.\nAborting" exit 1 else if [ -z "$IMAGE_GENERATION_DEBUG_MODE" ]; then echo -e "Empty string passed for debug mode. Invalid input. \nAborting" exit 1 fi fi if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" = "plain" ]; then echo "Impossible combination.\nAborting" exit 1 fi if [ "$PLUGIN" = "idh" -a "$HADOOP_VERSION" = "plain" ]; then echo "Impossible combination.\nAborting" exit 1 fi ################# if [ -e /etc/os-release ]; then platform=$(head -1 /etc/os-release) if [ $platform = 'NAME="Ubuntu"' ]; then apt-get update -y apt-get install qemu kpartx git -y elif [ $platform = 'NAME=Fedora' ]; then yum update -y yum install qemu kpartx git -y fi else platform=$(head -1 /etc/system-release | grep CentOS || :) if [ -n "$platform" ]; then yum update -y yum install qemu-kvm qemu-img kpartx git -y else echo -e "Unknown Host OS. Impossible to build images.\nAborting" exit 2 fi fi base_dir="$(dirname $(readlink -e $0))" TEMP=$(mktemp -d diskimage-create.XXXXXX) pushd $TEMP export DIB_IMAGE_CACHE=$TEMP/.cache-image-create # Working with repositories # disk-image-builder repo if [ -z $DIB_REPO_PATH ]; then git clone https://git.openstack.org/openstack/diskimage-builder DIB_REPO_PATH="$(pwd)/diskimage-builder" git --git-dir=$DIB_REPO_PATH/.git --work-tree=$DIB_REPO_PATH checkout $DIB_REPO_BRANCH fi export PATH=$PATH:$DIB_REPO_PATH/bin pushd $DIB_REPO_PATH export DIB_COMMIT_ID=`git rev-parse HEAD` popd export ELEMENTS_PATH="$DIB_REPO_PATH/elements" # sahara-image-elements repo if [ -z $SIM_REPO_PATH ]; then SIM_REPO_PATH="$(dirname $base_dir)" if [ $(basename $SIM_REPO_PATH) != "sahara-image-elements" ]; then echo "Can't find Sahara-image-elements repository. Cloning it." git clone https://git.openstack.org/openstack/sahara-image-elements SIM_REPO_PATH="$(pwd)/sahara-image-elements" fi fi ELEMENTS_PATH=$ELEMENTS_PATH:$SIM_REPO_PATH/elements pushd $SIM_REPO_PATH export SAHARA_ELEMENTS_COMMIT_ID=`git rev-parse HEAD` popd ############################# # Images for Vanilla plugin # ############################# if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"} export OOZIE_HADOOP_V1_DOWNLOAD_URL=${OOZIE_HADOOP_V1_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0.tar.gz"} export OOZIE_HADOOP_V2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0-hadoop-2.3.0.tar.gz"} export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"} export HIVE_VERSION=${HIVE_VERSION:-"0.11.0"} ubuntu_elements_sequence="base vm ubuntu hadoop oozie mysql hive" fedora_elements_sequence="base vm fedora hadoop oozie mysql hive disable-firewall" centos_elements_sequence="vm rhel hadoop oozie mysql hive redhat-lsb disable-firewall" # Workaround for https://bugs.launchpad.net/diskimage-builder/+bug/1204824 # https://bugs.launchpad.net/sahara/+bug/1252684 if [ -z "$IMAGE_TYPE" -o "$IMAGE_TYPE" = "centos" -o "$IMAGE_TYPE" = "fedora" ]; then if [ "$platform" = 'NAME="Ubuntu"' ]; then echo "**************************************************************" echo "WARNING: As a workaround for DIB bug 1204824, you are about to" echo " create a Fedora and CentOS images that has SELinux " echo " disabled. Do not use these images in production. " echo "**************************************************************" fedora_elements_sequence="$fedora_elements_sequence selinux-permissive" centos_elements_sequence="$centos_elements_sequence selinux-permissive" suffix=".selinux-permissive" fi fi if [ -n "$USE_MIRRORS" ]; then mirror_element=" apt-mirror" ubuntu_elements_sequence=$ubuntu_elements_sequence$mirror_element mirror_element=" yum-mirror" fedora_elements_sequence=$fedora_elements_sequence$mirror_element fi # Ubuntu cloud image if [ -z "$IMAGE_TYPE" -o "$IMAGE_TYPE" = "ubuntu" ]; then if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "1" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_1:-"1.2.1"} export ubuntu_image_name=${ubuntu_vanilla_hadoop_1_image_name:-"ubuntu_sahara_vanilla_hadoop_1_latest"} elements_sequence="$ubuntu_elements_sequence swift_hadoop" disk-image-create $elements_sequence -o $ubuntu_image_name mv $ubuntu_image_name.qcow2 ../ fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2:-"2.3.0"} export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_image_name:-"ubuntu_sahara_vanilla_hadoop_2_latest"} disk-image-create $ubuntu_elements_sequence -o $ubuntu_image_name mv $ubuntu_image_name.qcow2 ../ fi fi # Fedora cloud image if [ -z "$IMAGE_TYPE" -o "$IMAGE_TYPE" = "fedora" ]; then if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "1" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_1:-"1.2.1"} export fedora_image_name=${fedora_vanilla_hadoop_1_image_name:-"fedora_sahara_vanilla_hadoop_1_latest$suffix"} elements_sequence="$fedora_elements_sequence swift_hadoop" disk-image-create $elements_sequence -o $fedora_image_name mv $fedora_image_name.qcow2 ../ fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2:-"2.3.0"} export fedora_image_name=${fedora_vanilla_hadoop_2_image_name:-"fedora_sahara_vanilla_hadoop_2_latest$suffix"} disk-image-create $fedora_elements_sequence -o $fedora_image_name mv $fedora_image_name.qcow2 ../ fi fi # CentOS cloud image: # - Disable including 'base' element for CentOS # - Export link and filename for CentOS cloud image to download # - Patameter 'DIB_IMAGE_SIZE' should be specified for CentOS only if [ -z "$IMAGE_TYPE" -o "$IMAGE_TYPE" = "centos" ]; then export DIB_IMAGE_SIZE=${IMAGE_SIZE:-"10"} # Read Create_CentOS_cloud_image.rst to know how to create CentOS image in qcow2 format export BASE_IMAGE_FILE="CentOS-6.5-cloud-init.qcow2" export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "1" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_1:-"1.2.1"} export centos_image_name=${centos_vanilla_hadoop_1_image_name:-"centos_sahara_vanilla_hadoop_1_latest$suffix"} elements_sequence="$centos_elements_sequence swift_hadoop" disk-image-create $elements_sequence -n -o $centos_image_name mv $centos_image_name.qcow2 ../ fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2:-"2.3.0"} export centos_image_name=${centos_vanilla_hadoop_2_image_name:-"centos_sahara_vanilla_hadoop_2_latest$suffix"} disk-image-create $centos_elements_sequence -n -o $centos_image_name mv $centos_image_name.qcow2 ../ fi fi fi ########################## # Image for Spark plugin # ########################## if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then # Ignoring image type and hadoop version options echo "For spark plugin options -i and -v are ignored" unset DIB_IMAGE_SIZE export DIB_HADOOP_VERSION="2.0.0-mr1-cdh4.5.0" export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"} ubuntu_elements_sequence="base vm ubuntu hadoop-cdh spark" if [ -n "$USE_MIRRORS" ]; then mirror_element=" apt-mirror" ubuntu_elements_sequence=$ubuntu_elements_sequence$mirror_element fi # Creating Ubuntu cloud image disk-image-create $ubuntu_elements_sequence -o $ubuntu_image_name mv $ubuntu_image_name.qcow2 ../ fi ######################### # Images for HDP plugin # ######################### if [ -z "$PLUGIN" -o "$PLUGIN" = "hdp" ]; then echo "For hdp plugin option -i is ignored" # Generate HDP images # Parameter 'DIB_IMAGE_SIZE' should be specified for CentOS only export DIB_IMAGE_SIZE=${IMAGE_SIZE:-"10"} # CentOS cloud image: # - Disable including 'base' element for CentOS # - Export link and filename for CentOS cloud image to download export BASE_IMAGE_FILE="CentOS-6.4-cloud-init.qcow2" export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" if [ "$IMAGE_GENERATION_DEBUG_MODE" = "true" ]; then echo "Using HDP Image Debug Mode, using root-pwd in images, NOT FOR PRODUCTION USAGE." # Each image has a root login, password is "hadoop" export DIB_PASSWORD="hadoop" fi # Ignoring image type option if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "1" ]; then export centos_image_name_hdp_1_3=${centos_hdp_hadoop_1_image_name:-"centos-6_4-64-hdp-1-3"} # Elements to include in an HDP-based image centos_elements_sequence="vm rhel hadoop-hdp disable-firewall redhat-lsb sahara-version source-repositories yum disable-selinux" if [ "$IMAGE_GENERATION_DEBUG_MODE" = "true" ]; then # enable the root-pwd element, for simpler local debugging of images centos_elements_sequence=$centos_elements_sequence" root-passwd" fi # generate image with HDP 1.3 export DIB_HDP_VERSION="1.3" disk-image-create $centos_elements_sequence -n -o $centos_image_name_hdp_1_3 mv $centos_image_name_hdp_1_3.qcow2 ../ fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2" ]; then export centos_image_name_hdp_2_0=${centos_hdp_hadoop_2_image_name:-"centos-6_4-64-hdp-2-0"} # Elements to include in an HDP-based image centos_elements_sequence="vm rhel hadoop-hdp disable-firewall redhat-lsb sahara-version source-repositories yum disable-selinux" if [ "$IMAGE_GENERATION_DEBUG_MODE" = "true" ]; then # enable the root-pwd element, for simpler local debugging of images centos_elements_sequence=$centos_elements_sequence" root-passwd" fi # generate image with HDP 2.0 export DIB_HDP_VERSION="2.0" disk-image-create $centos_elements_sequence -n -o $centos_image_name_hdp_2_0 mv $centos_image_name_hdp_2_0.qcow2 ../ fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "plain" ]; then export centos_image_name_plain=${centos_hdp_plain_image_name:-"centos-6_4-64-plain"} # Elements for a plain CentOS image that does not contain HDP or Apache Hadoop centos_plain_elements_sequence="vm rhel redhat-lsb sahara-version yum disable-selinux" if [ "$IMAGE_GENERATION_DEBUG_MODE" = "true" ]; then # enable the root-pwd element, for simpler local debugging of images centos_elements_sequence=$centos_elements_sequence" root-passwd" fi # generate plain (no Hadoop components) image for testing disk-image-create $centos_plain_elements_sequence -n -o $centos_image_name_plain mv $centos_image_name_plain.qcow2 ../ fi fi ######################## # Image for IDH plugin # ######################## if [ -z "$PLUGIN" -o "$PLUGIN" = "idh" ]; then # Ignoring image type and hadoop version options echo "For idh plugin option -i is ignored" export DIB_IMAGE_SIZE=${IMAGE_SIZE:-"10"} export BASE_IMAGE_FILE="CentOS-6.4-cloud-init.qcow2" export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" centos_elements_sequence="vm rhel hadoop-idh" if [ "$platform" = 'NAME="Ubuntu"' ]; then echo "**************************************************************" echo "WARNING: As a workaround for DIB bug 1204824, you are about to" echo " create a CentOS image that has SELinux " echo " disabled. Do not use these images in production. " echo "**************************************************************" centos_elements_sequence="$centos_elements_sequence selinux-permissive" suffix=".selinux-permissive" fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "1" ]; then export HADOOP_VERSION="1" export DIB_HADOOP_VERSION="2.5.1" export centos_image_name_idh="centos_sahara_idh_${DIB_HADOOP_VERSION}${suffix}" disk-image-create $centos_elements_sequence -o $centos_image_name_idh mv $centos_image_name_idh.qcow2 ../ fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2" ]; then export HADOOP_VERSION="2" export DIB_HADOOP_VERSION="3.0.2" export centos_image_name_idh="centos_sahara_idh_${DIB_HADOOP_VERSION}${suffix}" disk-image-create $centos_elements_sequence -o $centos_image_name_idh mv $centos_image_name_idh.qcow2 ../ fi fi popd # out of $TEMP rm -rf $TEMP