Fixing spark and CDH refinements
Actually there is no much difference between using CDH 5.4 and 5.5 for spark, because both are using Hadoop 2.6 as a base. Also removing redundant case in installing packages for CDH. Change-Id: Ie24bb72365352edb22d94a461df0a0af6cd71806 Closes-bug: 1686400
This commit is contained in:
parent
7b1e5fb03d
commit
972ce02c76
@ -200,19 +200,6 @@ case "$PLUGIN" in
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$HADOOP_VERSION" in
|
||||
"")
|
||||
echo "CDH version not specified"
|
||||
echo "CDH version 5.5 will be used"
|
||||
HADOOP_VERSION="5.5"
|
||||
;;
|
||||
"5.5");;
|
||||
*)
|
||||
echo -e "Unknown hadoop version selected.\nAborting"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$DIB_SPARK_VERSION" in
|
||||
"1.3.1" | "1.6.0");;
|
||||
"")
|
||||
@ -551,19 +538,11 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
|
||||
export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
|
||||
export DIB_SPARK_VERSION
|
||||
export plugin_type="spark"
|
||||
|
||||
COMMON_ELEMENTS="$JAVA_ELEMENT swift_hadoop spark"
|
||||
if [ "$DIB_SPARK_VERSION" == "1.6.0" ]; then
|
||||
export DIB_CDH_VERSION="5.4"
|
||||
ubuntu_elements_sequence="$COMMON_ELEMENTS hadoop-cloudera"
|
||||
else
|
||||
export DIB_CDH_VERSION=$HADOOP_VERSION
|
||||
ubuntu_elements_sequence="$COMMON_ELEMENTS hadoop-cloudera"
|
||||
fi
|
||||
|
||||
export DIB_CDH_VERSION="5.5"
|
||||
# Tell the cloudera element to install only hdfs
|
||||
export DIB_CDH_HDFS_ONLY=1
|
||||
|
||||
ubuntu_elements_sequence="$JAVA_ELEMENT swift_hadoop spark hadoop-cloudera"
|
||||
export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"}
|
||||
|
||||
# Creating Ubuntu cloud image
|
||||
|
@ -31,10 +31,7 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
|
||||
hue \
|
||||
oozie \
|
||||
spark-core \
|
||||
zookeeper
|
||||
|
||||
if [ $DIB_CDH_VERSION \> "5.0" ]; then
|
||||
install-packages \
|
||||
zookeeper \
|
||||
flume-ng \
|
||||
hadoop-kms \
|
||||
hbase-solr \
|
||||
@ -53,7 +50,6 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
|
||||
kafka \
|
||||
kafka-server
|
||||
fi
|
||||
fi
|
||||
|
||||
DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0}
|
||||
|
||||
|
@ -15,7 +15,7 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then
|
||||
# INFO on hadoop versions: http://spark.apache.org/docs/latest/hadoop-third-party-distributions.html
|
||||
if [ -z "${SPARK_HADOOP_DL:-}" ]; then
|
||||
case "${DIB_CDH_VERSION:-}" in
|
||||
5.4)
|
||||
5.5)
|
||||
SPARK_HADOOP_DL=hadoop2.6
|
||||
;;
|
||||
*)
|
||||
|
Loading…
Reference in New Issue
Block a user