openstack/sahara-image-elements
Code Issues Proposed changes

Configurable Spark version for Vanilla image

Browse Source 
Instead of hard-coded Spark 1.6.0, allow use of DIB_SPARK_VERSION (and
its corresponding CLI argument `-s`) to specify which version of Spark
should be included on images for Vanilla plugin.

Change-Id: Ia7c7027c9eadfc9d724733a503990ca78e487ee9
This commit is contained in:
Jeremy Freudberg 2017-07-17 19:25:37 +00:00
parent 3425012a84
commit ca8767104b
2 changed files with 32 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
diskimage-create/README.rst
View File

@ -80,7 +80,8 @@ like this:
.. sourcecode:: bash .. sourcecode:: bash
tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0] tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0] # spark standalone
tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0] # spark on vanilla
8. To select which MapR version to target use the '-r' commandline option like 8. To select which MapR version to target use the '-r' commandline option like
this: this:

44
diskimage-create/diskimage-create.sh
View File

@ -160,6 +160,21 @@ if [ "$BASE_IMAGE_OS" = "centos" ]; then
echo "*************************************************" echo "*************************************************"
fi fi
check_spark_version () {
case "$DIB_SPARK_VERSION" in
"1.3.1" | "1.6.0" | "2.1.0");;
"")
echo "Spark version not specified"
echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used"
DIB_SPARK_VERSION=${DIB_DEFAULT_SPARK_VERSION}
;;
*)
echo -e "Unknown Spark version selected.\nAborting"
exit 1
;;
esac
}
case "$PLUGIN" in case "$PLUGIN" in
"");; "");;
"vanilla") "vanilla")
@ -177,6 +192,7 @@ case "$PLUGIN" in
exit 1 exit 1
;; ;;
esac esac
check_spark_version
;; ;;
"cloudera") "cloudera")
case "$BASE_IMAGE_OS" in case "$BASE_IMAGE_OS" in
@ -214,20 +230,7 @@ case "$PLUGIN" in
exit 1 exit 1
;; ;;
esac esac
check_spark_version
case "$DIB_SPARK_VERSION" in
"1.3.1" | "1.6.0" | "2.1.0");;
"")
echo "Spark version not specified"
echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used"
DIB_SPARK_VERSION=${DIB_DEFAULT_SPARK_VERSION}
;;
*)
echo -e "Unknown Spark version selected.\nAborting"
exit 1
;;
esac
;; ;;
"storm") "storm")
case "$BASE_IMAGE_OS" in case "$BASE_IMAGE_OS" in
@ -476,8 +479,19 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"http://sahara-files.mirantis.com/oozie-4.2.0-hadoop-2.7.1.tar.gz"} export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"http://sahara-files.mirantis.com/oozie-4.2.0-hadoop-2.7.1.tar.gz"}
export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib" export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib"
export plugin_type="vanilla" export plugin_type="vanilla"
export DIB_SPARK_VERSION=1.6.0
if [ "$DIB_SPARK_VERSION" = "1.3.1" ]; then
echo "Cannot use Spark 1.3.1 with Vanilla plugin"
exit 1
fi
export DIB_SPARK_VERSION
if [ "$DIB_SPARK_VERSION" = "1.6.0" ]; then
export SPARK_HADOOP_DL=hadoop2.6 export SPARK_HADOOP_DL=hadoop2.6
else
export SPARK_HADOOP_DL=hadoop2.7
fi
ubuntu_elements_sequence="hadoop oozie mysql hive $JAVA_ELEMENT swift_hadoop spark" ubuntu_elements_sequence="hadoop oozie mysql hive $JAVA_ELEMENT swift_hadoop spark"
fedora_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark" fedora_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark"