diff --git a/diskimage-create/README.rst b/diskimage-create/README.rst index 0c3d1e1f..ca815b84 100644 --- a/diskimage-create/README.rst +++ b/diskimage-create/README.rst @@ -80,8 +80,8 @@ like this: .. sourcecode:: bash - tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0] # spark standalone - tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0] # spark on vanilla + tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0|2.2.0] # spark standalone + tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0|2.2.0] # spark on vanilla 8. To select which MapR version to target use the '-r' commandline option like this: diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index a8edcf60..5b3693b4 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -13,7 +13,7 @@ DEBUG_MODE="false" DIB_DEFAULT_MAPR_VERSION="5.2.0" # The default version for Spark plugin -DIB_DEFAULT_SPARK_VERSION="2.1.0" +DIB_DEFAULT_SPARK_VERSION="2.2.0" # The default version for Storm plugin DIB_DEFAULT_STORM_VERSION="1.1.0" @@ -34,7 +34,7 @@ usage() { echo " [-i ubuntu|fedora|centos|centos7]" echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]" echo " [-r 5.1.0|5.2.0]" - echo " [-s 1.3.1|1.6.0|2.1.0]" + echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]" echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]" echo " [-f qcow2|raw]" echo " [-d]" @@ -167,7 +167,7 @@ fi check_spark_version () { case "$DIB_SPARK_VERSION" in - "1.3.1" | "1.6.0" | "2.1.0");; + "1.3.1" | "1.6.0" | "2.1.0" | "2.2.0");; "") echo "Spark version not specified" echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used" @@ -563,16 +563,23 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then unset SPARK_HADOOP_DL fi -########################## -# Image for Spark plugin # -########################## +########################### +# Images for Spark plugin # +########################### if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce" export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES export DIB_SPARK_VERSION export plugin_type="spark" - export DIB_CDH_VERSION="5.5" + + if [ "$DIB_SPARK_VERSION" = "2.2.0" ]; then + export DIB_CDH_VERSION="5.11" + export DIB_RELEASE=${DIB_RELEASE:-xenial} + else + export DIB_RELEASE=${DIB_RELEASE:-trusty} + export DIB_CDH_VERSION="5.5" + fi # Tell the cloudera element to install only hdfs export DIB_CDH_HDFS_ONLY=1 @@ -580,7 +587,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"} # Creating Ubuntu cloud image - export DIB_RELEASE=${DIB_RELEASE:-trusty} image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence unset DIB_CLOUD_INIT_DATASOURCES unset DIB_HDFS_LIB_DIR diff --git a/elements/spark/root.d/50-download-spark b/elements/spark/root.d/50-download-spark index 757bcbce..eac5b47d 100755 --- a/elements/spark/root.d/50-download-spark +++ b/elements/spark/root.d/50-download-spark @@ -18,6 +18,9 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then 5.5) SPARK_HADOOP_DL=hadoop2.6 ;; + 5.11) + SPARK_HADOOP_DL=hadoop2.6 + ;; *) echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported." echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version."