Adding Spark 2.2.0
Adding newest version of spark. Change-Id: Ib2894d5d93d3ecfd17e0fb1eba4d687a97406027
This commit is contained in:
parent
589abcce31
commit
05085a81db
|
@ -80,8 +80,8 @@ like this:
|
||||||
|
|
||||||
.. sourcecode:: bash
|
.. sourcecode:: bash
|
||||||
|
|
||||||
tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0] # spark standalone
|
tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0|2.2.0] # spark standalone
|
||||||
tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0] # spark on vanilla
|
tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0|2.2.0] # spark on vanilla
|
||||||
|
|
||||||
8. To select which MapR version to target use the '-r' commandline option like
|
8. To select which MapR version to target use the '-r' commandline option like
|
||||||
this:
|
this:
|
||||||
|
|
|
@ -13,7 +13,7 @@ DEBUG_MODE="false"
|
||||||
DIB_DEFAULT_MAPR_VERSION="5.2.0"
|
DIB_DEFAULT_MAPR_VERSION="5.2.0"
|
||||||
|
|
||||||
# The default version for Spark plugin
|
# The default version for Spark plugin
|
||||||
DIB_DEFAULT_SPARK_VERSION="2.1.0"
|
DIB_DEFAULT_SPARK_VERSION="2.2.0"
|
||||||
|
|
||||||
# The default version for Storm plugin
|
# The default version for Storm plugin
|
||||||
DIB_DEFAULT_STORM_VERSION="1.1.0"
|
DIB_DEFAULT_STORM_VERSION="1.1.0"
|
||||||
|
@ -34,7 +34,7 @@ usage() {
|
||||||
echo " [-i ubuntu|fedora|centos|centos7]"
|
echo " [-i ubuntu|fedora|centos|centos7]"
|
||||||
echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
|
echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
|
||||||
echo " [-r 5.1.0|5.2.0]"
|
echo " [-r 5.1.0|5.2.0]"
|
||||||
echo " [-s 1.3.1|1.6.0|2.1.0]"
|
echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
|
||||||
echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
|
echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
|
||||||
echo " [-f qcow2|raw]"
|
echo " [-f qcow2|raw]"
|
||||||
echo " [-d]"
|
echo " [-d]"
|
||||||
|
@ -167,7 +167,7 @@ fi
|
||||||
|
|
||||||
check_spark_version () {
|
check_spark_version () {
|
||||||
case "$DIB_SPARK_VERSION" in
|
case "$DIB_SPARK_VERSION" in
|
||||||
"1.3.1" | "1.6.0" | "2.1.0");;
|
"1.3.1" | "1.6.0" | "2.1.0" | "2.2.0");;
|
||||||
"")
|
"")
|
||||||
echo "Spark version not specified"
|
echo "Spark version not specified"
|
||||||
echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used"
|
echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used"
|
||||||
|
@ -563,16 +563,23 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
|
||||||
unset SPARK_HADOOP_DL
|
unset SPARK_HADOOP_DL
|
||||||
fi
|
fi
|
||||||
|
|
||||||
##########################
|
###########################
|
||||||
# Image for Spark plugin #
|
# Images for Spark plugin #
|
||||||
##########################
|
###########################
|
||||||
|
|
||||||
if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
|
if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
|
||||||
export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce"
|
export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce"
|
||||||
export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
|
export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
|
||||||
export DIB_SPARK_VERSION
|
export DIB_SPARK_VERSION
|
||||||
export plugin_type="spark"
|
export plugin_type="spark"
|
||||||
export DIB_CDH_VERSION="5.5"
|
|
||||||
|
if [ "$DIB_SPARK_VERSION" = "2.2.0" ]; then
|
||||||
|
export DIB_CDH_VERSION="5.11"
|
||||||
|
export DIB_RELEASE=${DIB_RELEASE:-xenial}
|
||||||
|
else
|
||||||
|
export DIB_RELEASE=${DIB_RELEASE:-trusty}
|
||||||
|
export DIB_CDH_VERSION="5.5"
|
||||||
|
fi
|
||||||
# Tell the cloudera element to install only hdfs
|
# Tell the cloudera element to install only hdfs
|
||||||
export DIB_CDH_HDFS_ONLY=1
|
export DIB_CDH_HDFS_ONLY=1
|
||||||
|
|
||||||
|
@ -580,7 +587,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
|
||||||
export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"}
|
export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"}
|
||||||
|
|
||||||
# Creating Ubuntu cloud image
|
# Creating Ubuntu cloud image
|
||||||
export DIB_RELEASE=${DIB_RELEASE:-trusty}
|
|
||||||
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
|
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
|
||||||
unset DIB_CLOUD_INIT_DATASOURCES
|
unset DIB_CLOUD_INIT_DATASOURCES
|
||||||
unset DIB_HDFS_LIB_DIR
|
unset DIB_HDFS_LIB_DIR
|
||||||
|
|
|
@ -18,6 +18,9 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then
|
||||||
5.5)
|
5.5)
|
||||||
SPARK_HADOOP_DL=hadoop2.6
|
SPARK_HADOOP_DL=hadoop2.6
|
||||||
;;
|
;;
|
||||||
|
5.11)
|
||||||
|
SPARK_HADOOP_DL=hadoop2.6
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported."
|
echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported."
|
||||||
echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version."
|
echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version."
|
||||||
|
|
Loading…
Reference in New Issue