openstack/sahara-image-elements
Code Issues Proposed changes

Add support to build 2.8.2 image

Browse Source 
Also tweak Hive a bit and refer to artifacts in a new (but not totally
ideal) location.

Co-Authored-By: Jeremy Freudberg <jeremyfreudberg@gmail.com>
Change-Id: I3a25ee8c282849911089adf6c3593b1bb50fd067
This commit is contained in:
Shu Yingya 2018-02-15 17:16:40 +01:00
parent 8cdff5d71f
commit 12efcf84e2
10 changed files with 92 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
diskimage-create/diskimage-create.sh
View File

@ -32,7 +32,7 @@ usage() {
echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
echo " [-i ubuntu|fedora|centos|centos7]"
echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-v 2.7.1|2.8.2|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-r 5.1.0|5.2.0]"
echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
@ -184,7 +184,7 @@ case "$PLUGIN" in
"");;
"vanilla")
case "$HADOOP_VERSION" in
"" | "2.7.1");;
"" | "2.7.1" | "2.8.2");;
*)
echo -e "Unknown hadoop version selected.\nAborting"
exit 1
@ -474,14 +474,33 @@ image_create() {
esac
}
set_hive_version() {
if [ -z "${HIVE_VERSION:-}" ]; then
case "$DIB_HADOOP_VERSION" in
"2.7.1" )
export HIVE_VERSION="0.11.0"
;;
"2.8.2" )
export HIVE_VERSION="2.3.2"
;;
*)
echo -e "Unknown Hadoop version, therefore cannot choose Hive version.\nAborting."
exit 1
;;
esac
fi
}
#############################
# Images for Vanilla plugin #
#############################
if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
export HIVE_VERSION=${HIVE_VERSION:-"0.11.0"}
export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"}
export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"}
export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"}
export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"}
export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib"
export plugin_type="vanilla"
@ -524,6 +543,15 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_1_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_1_latest"}
export DIB_RELEASE=${DIB_RELEASE:-xenial}
set_hive_version
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
unset DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_8_2_image_name:-"ubuntu_sahara_vanilla_hadoop_2_8_2_latest"}
export DIB_RELEASE=${DIB_RELEASE:-xenial}
set_hive_version
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
unset DIB_RELEASE
fi
@ -535,6 +563,13 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
export fedora_image_name=${fedora_vanilla_hadoop_2_7_1_image_name:-"fedora_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
set_hive_version
image_create fedora $fedora_image_name $fedora_elements_sequence
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
export fedora_image_name=${fedora_vanilla_hadoop_2_8_2_image_name:-"fedora_sahara_vanilla_hadoop_2_8_2_latest$suffix"}
set_hive_version
image_create fedora $fedora_image_name $fedora_elements_sequence
fi
fi
@ -544,6 +579,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
export centos_image_name=${centos_vanilla_hadoop_2_7_1_image_name:-"centos_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
set_hive_version
image_create centos $centos_image_name $centos_elements_sequence
fi
fi
@ -553,6 +589,13 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
export centos7_image_name=${centos7_vanilla_hadoop_2_7_1_image_name:-"centos7_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
set_hive_version
image_create centos7 $centos7_image_name $centos7_elements_sequence
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
export centos7_image_name=${centos7_vanilla_hadoop_2_8_2_image_name:-"centos7_sahara_vanilla_hadoop_2_8_2_latest$suffix"}
set_hive_version
image_create centos7 $centos7_image_name $centos7_elements_sequence
fi
fi

3
elements/hadoop/install.d/40-setup-hadoop
View File

@ -11,6 +11,9 @@ function install_hadoop_v2 {
"2.7.1")
hadoop_native_libs_url="${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL}"
;;
"2.8.2")
hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}"
;;
*)
echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION"
exit 1

16
elements/hive/post-install.d/60-hive
View File

@ -10,7 +10,12 @@ tmp_dir=/tmp/hive
cd $tmp_dir
if [ -z "${HIVE_DOWNLOAD_URL:-}" ]; then
HIVE_FILE=hive-$HIVE_VERSION-bin.tar.gz
# hive package has renamed to "apache-hive-*" since 0.13.0
if [[ "$HIVE_VERSION" < "0.13.0" ]]; then
HIVE_FILE=hive-$HIVE_VERSION-bin.tar.gz
else
HIVE_FILE=apache-hive-$HIVE_VERSION-bin.tar.gz
fi
else
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
fi
@ -21,6 +26,15 @@ mv $HIVE_DIR /opt/hive
rm -r $tmp_dir
chmod -R a+rw /opt/hive
HIVE_MASTER_VERSION=${HIVE_VERSION%.*}.0
# NOTE(jfreud): Hive 0.11.0 was released without this DB upgrade script.
# All future versions of Hive do have the script.
if [ "$HIVE_VERSION" != "0.11.0" ]; then
# replace the relative path with absolute path
sed -i "s/SOURCE hive-txn-schema-$HIVE_MASTER_VERSION.mysql.sql;/SOURCE \/opt\/hive\/scripts\/metastore\/upgrade\/mysql\/hive-txn-schema-$HIVE_MASTER_VERSION.mysql.sql;/" /opt/hive/scripts/metastore/upgrade/mysql/hive-schema-$HIVE_MASTER_VERSION.mysql.sql
fi
ln -s /usr/share/java/mysql-connector-java.jar /opt/hive/lib/libmysql-java.jar
chown -R hadoop:hadoop /opt/hive
cat >> /home/hadoop/.bashrc <<EOF

2
elements/hive/root.d/0-check-hive
View File

@ -7,7 +7,7 @@ set -eu
set -o pipefail
if [ -z "${HIVE_DOWNLOAD_URL:-}" ]; then
version_check=$(echo $HIVE_VERSION | sed -e '/[0-9]\.[0-9][0-9]\.[0-9]/d')
version_check=$(echo $HIVE_VERSION | sed -e '/[0-9]\.[0-9]\{1,2\}\.[0-9]/d')
if [ ! -z $version_check ]; then
echo -e "Unable to install Hive: You should specify HIVE_DOWNLOAD_URL or HIVE_VERSION.\nAborting"
exit 1

8
elements/hive/root.d/50-download-hive
View File

@ -10,7 +10,13 @@ tmp_dir=$TARGET_ROOT/tmp/hive
mkdir -p $tmp_dir
if [ -z ${HIVE_DOWNLOAD_URL:-} ]; then
HIVE_DOWNLOAD_URL=http://archive.apache.org/dist/hive/hive-$HIVE_VERSION/hive-$HIVE_VERSION-bin.tar.gz
# hive package has renamed to "apache-hive-*" since 0.13.0
if [[ "$HIVE_VERSION" < "0.13.0" ]]; then
HIVE_FILE=hive-$HIVE_VERSION-bin.tar.gz
else
HIVE_FILE=apache-hive-$HIVE_VERSION-bin.tar.gz
fi
HIVE_DOWNLOAD_URL=http://archive.apache.org/dist/hive/hive-$HIVE_VERSION/$HIVE_FILE
fi
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
cached_tar="$DIB_IMAGE_CACHE/$HIVE_FILE"

6
elements/oozie/install.d/50-setup-oozie
View File

@ -16,6 +16,12 @@ if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then
else
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL)
fi
elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" ]; then
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_FILE)
else
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL)
fi
else
echo "Unknown Hadoop version selected. Aborting."
exit 1

5
elements/oozie/root.d/0-check-oozie
View File

@ -11,6 +11,11 @@ if [ "$DIB_HADOOP_VERSION" == "2.7.1" ]; then
echo "OOZIE_HADOOP_V2_7_1_FILE and OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
exit 1
fi
elif [ "$DIB_HADOOP_VERSION" == "2.8.2" ]; then
if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_8_2_FILE:-}" ]; then
echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
exit 1
fi
else
echo "Unknown Hadoop version selected. Aborting"
exit 1

3
elements/oozie/root.d/50-download-oozie
View File

@ -12,6 +12,9 @@ mkdir -p $tmp_dir
if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V2_7_1_FILE:-}"
elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}"
OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}"
else
echo "Unknown Hadoop version selected. Aborting."
exit 1

2
elements/swift_hadoop/post-install.d/81-add-jar
View File

@ -12,6 +12,8 @@ case "$plugin_type" in
case "$DIB_HADOOP_VERSION" in
"2.7.1")
hadoop="2.7.1";;
"2.8.2")
hadoop="2.8.2";;
esac
DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop.jar"
;;

6
tools/gate/build-images
View File

@ -6,8 +6,10 @@ case "$PLUGIN" in
"vanilla")
# temporary don't test building fedora image
for distro in ubuntu centos7; do
# testing only 2.7.1 for speed up
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v 2.7.1
# limit to 2 releases
for version in 2.7.1 2.8.2; do
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version
done
done
;;
"cloudera")